2024-11-18 23:15:06,393 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 23:15:06,408 main DEBUG Took 0.012952 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 23:15:06,409 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 23:15:06,409 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 23:15:06,411 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 23:15:06,412 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,422 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 23:15:06,440 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,444 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,446 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,447 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,449 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,451 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,452 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,452 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,455 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,457 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,458 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,459 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 23:15:06,460 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,460 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 23:15:06,462 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 23:15:06,464 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 23:15:06,467 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 23:15:06,468 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 23:15:06,470 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 23:15:06,470 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 23:15:06,481 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 23:15:06,484 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 23:15:06,487 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 23:15:06,487 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 23:15:06,487 main DEBUG createAppenders(={Console}) 2024-11-18 23:15:06,488 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-18 23:15:06,489 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-18 23:15:06,489 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-18 23:15:06,490 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 23:15:06,490 main DEBUG OutputStream closed 2024-11-18 23:15:06,491 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 23:15:06,491 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 23:15:06,491 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-18 23:15:06,587 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 23:15:06,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 23:15:06,592 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 23:15:06,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 23:15:06,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 23:15:06,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 23:15:06,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 23:15:06,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 23:15:06,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 23:15:06,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 23:15:06,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 23:15:06,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 23:15:06,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 23:15:06,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 23:15:06,598 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 23:15:06,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 23:15:06,599 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 23:15:06,600 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 23:15:06,603 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 23:15:06,603 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-18 23:15:06,603 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 23:15:06,604 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-18T23:15:06,866 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f 2024-11-18 23:15:06,870 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 23:15:06,870 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T23:15:06,882 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-18T23:15:06,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-18T23:15:06,937 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771, deleteOnExit=true 2024-11-18T23:15:06,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-18T23:15:06,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/test.cache.data in system properties and HBase conf 2024-11-18T23:15:06,940 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/hadoop.tmp.dir in system properties and HBase conf 2024-11-18T23:15:06,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/hadoop.log.dir in system properties and HBase conf 2024-11-18T23:15:06,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-18T23:15:06,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-18T23:15:06,944 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-18T23:15:07,091 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T23:15:07,230 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T23:15:07,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-18T23:15:07,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-18T23:15:07,245 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-18T23:15:07,246 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T23:15:07,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-18T23:15:07,247 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-18T23:15:07,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-18T23:15:07,248 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T23:15:07,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-18T23:15:07,249 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/nfs.dump.dir in system properties and HBase conf 2024-11-18T23:15:07,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/java.io.tmpdir in system properties and HBase conf 2024-11-18T23:15:07,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-18T23:15:07,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-18T23:15:07,252 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-18T23:15:08,619 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-18T23:15:08,710 INFO [Time-limited test {}] log.Log(170): Logging initialized @3323ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-18T23:15:08,800 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:15:08,879 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T23:15:08,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T23:15:08,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T23:15:08,917 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-18T23:15:08,928 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:15:08,932 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/hadoop.log.dir/,AVAILABLE} 2024-11-18T23:15:08,933 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T23:15:09,202 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/java.io.tmpdir/jetty-localhost-36907-hadoop-hdfs-3_4_1-tests_jar-_-any-18022712891295984636/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T23:15:09,218 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:36907} 2024-11-18T23:15:09,218 INFO [Time-limited test {}] server.Server(415): Started @3832ms 2024-11-18T23:15:10,087 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-18T23:15:10,095 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-18T23:15:10,097 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-18T23:15:10,097 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-18T23:15:10,097 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-18T23:15:10,098 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/hadoop.log.dir/,AVAILABLE} 2024-11-18T23:15:10,099 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-18T23:15:10,223 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/java.io.tmpdir/jetty-localhost-41889-hadoop-hdfs-3_4_1-tests_jar-_-any-110862271031494429/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:15:10,223 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:41889} 2024-11-18T23:15:10,224 INFO [Time-limited test {}] server.Server(415): Started @4837ms 2024-11-18T23:15:10,273 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-18T23:15:11,370 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/data/data2/current/BP-2062329126-172.17.0.2-1731971708035/current, will proceed with Du for space computation calculation, 2024-11-18T23:15:11,370 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/data/data1/current/BP-2062329126-172.17.0.2-1731971708035/current, will proceed with Du for space computation calculation, 2024-11-18T23:15:11,403 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-18T23:15:11,448 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1876c521dae85f06 with lease ID 0x2c61f7c6ceb3413d: Processing first storage report for DS-2758df84-89f8-46db-af55-fa2cd38e60fe from datanode DatanodeRegistration(127.0.0.1:43067, datanodeUuid=240527c7-c924-4f5d-b701-50d98ed6d905, infoPort=39723, infoSecurePort=0, ipcPort=43557, storageInfo=lv=-57;cid=testClusterID;nsid=1681302402;c=1731971708035) 2024-11-18T23:15:11,449 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1876c521dae85f06 with lease ID 0x2c61f7c6ceb3413d: from storage DS-2758df84-89f8-46db-af55-fa2cd38e60fe node DatanodeRegistration(127.0.0.1:43067, datanodeUuid=240527c7-c924-4f5d-b701-50d98ed6d905, infoPort=39723, infoSecurePort=0, ipcPort=43557, storageInfo=lv=-57;cid=testClusterID;nsid=1681302402;c=1731971708035), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-18T23:15:11,450 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1876c521dae85f06 with lease ID 0x2c61f7c6ceb3413d: Processing first storage report for DS-e70e26aa-b021-439c-a1db-8cc7f550930f from datanode DatanodeRegistration(127.0.0.1:43067, datanodeUuid=240527c7-c924-4f5d-b701-50d98ed6d905, infoPort=39723, infoSecurePort=0, ipcPort=43557, storageInfo=lv=-57;cid=testClusterID;nsid=1681302402;c=1731971708035) 2024-11-18T23:15:11,450 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1876c521dae85f06 with lease ID 0x2c61f7c6ceb3413d: from storage DS-e70e26aa-b021-439c-a1db-8cc7f550930f node DatanodeRegistration(127.0.0.1:43067, datanodeUuid=240527c7-c924-4f5d-b701-50d98ed6d905, infoPort=39723, infoSecurePort=0, ipcPort=43557, storageInfo=lv=-57;cid=testClusterID;nsid=1681302402;c=1731971708035), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-18T23:15:11,469 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f 2024-11-18T23:15:11,557 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/zookeeper_0, clientPort=56682, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T23:15:11,572 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56682 2024-11-18T23:15:11,588 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:15:11,593 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:15:11,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741825_1001 (size=7) 2024-11-18T23:15:12,312 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff with version=8 2024-11-18T23:15:12,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/hbase-staging 2024-11-18T23:15:12,428 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T23:15:12,672 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9d96596e55a9:0 server-side Connection retries=45 2024-11-18T23:15:12,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:15:12,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:15:12,686 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:15:12,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:15:12,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:15:12,845 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T23:15:12,935 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T23:15:12,949 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T23:15:12,954 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:15:12,991 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3564 (auto-detected) 2024-11-18T23:15:12,993 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T23:15:13,026 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42339 2024-11-18T23:15:13,057 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42339 connecting to ZooKeeper ensemble=127.0.0.1:56682 2024-11-18T23:15:13,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423390x0, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:15:13,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42339-0x101503afdd90000 connected 2024-11-18T23:15:13,262 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:15:13,265 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:15:13,276 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:15:13,281 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff, hbase.cluster.distributed=false 2024-11-18T23:15:13,307 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:15:13,312 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42339 2024-11-18T23:15:13,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42339 2024-11-18T23:15:13,313 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42339 2024-11-18T23:15:13,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42339 2024-11-18T23:15:13,314 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42339 2024-11-18T23:15:13,426 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9d96596e55a9:0 server-side Connection retries=45 2024-11-18T23:15:13,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:15:13,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T23:15:13,428 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T23:15:13,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T23:15:13,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T23:15:13,430 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T23:15:13,433 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T23:15:13,434 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32899 2024-11-18T23:15:13,436 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32899 connecting to ZooKeeper ensemble=127.0.0.1:56682 2024-11-18T23:15:13,437 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:15:13,440 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:15:13,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328990x0, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T23:15:13,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328990x0, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:15:13,450 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32899-0x101503afdd90001 connected 2024-11-18T23:15:13,454 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T23:15:13,461 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T23:15:13,463 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T23:15:13,468 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T23:15:13,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32899 2024-11-18T23:15:13,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32899 2024-11-18T23:15:13,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32899 2024-11-18T23:15:13,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32899 2024-11-18T23:15:13,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32899 2024-11-18T23:15:13,485 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9d96596e55a9:42339 2024-11-18T23:15:13,486 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9d96596e55a9,42339,1731971712494 2024-11-18T23:15:13,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:15:13,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:15:13,504 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9d96596e55a9,42339,1731971712494 2024-11-18T23:15:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T23:15:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:13,535 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T23:15:13,536 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9d96596e55a9,42339,1731971712494 from backup master directory 2024-11-18T23:15:13,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:15:13,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9d96596e55a9,42339,1731971712494 2024-11-18T23:15:13,544 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T23:15:13,545 WARN [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:15:13,545 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9d96596e55a9,42339,1731971712494 2024-11-18T23:15:13,547 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T23:15:13,548 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T23:15:13,599 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/hbase.id] with ID: 312958eb-d8b4-4384-a210-556b66aba6db 2024-11-18T23:15:13,599 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/.tmp/hbase.id 2024-11-18T23:15:13,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741826_1002 (size=42) 2024-11-18T23:15:14,011 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/.tmp/hbase.id]:[hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/hbase.id] 2024-11-18T23:15:14,057 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-18T23:15:14,062 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T23:15:14,080 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-18T23:15:14,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:14,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:14,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741827_1003 (size=196) 2024-11-18T23:15:14,524 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:15:14,527 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T23:15:14,542 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:14,546 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:15:14,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741828_1004 (size=1189) 2024-11-18T23:15:14,594 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store 2024-11-18T23:15:14,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741829_1005 (size=34) 2024-11-18T23:15:15,018 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T23:15:15,022 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:15,023 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:15:15,023 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:15:15,023 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:15:15,025 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:15:15,025 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:15:15,026 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:15:15,027 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731971715023Disabling compacts and flushes for region at 1731971715023Disabling writes for close at 1731971715025 (+2 ms)Writing region close event to WAL at 1731971715026 (+1 ms)Closed at 1731971715026 2024-11-18T23:15:15,030 WARN [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/.initializing 2024-11-18T23:15:15,030 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/WALs/9d96596e55a9,42339,1731971712494 2024-11-18T23:15:15,040 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:15:15,055 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C42339%2C1731971712494, suffix=, logDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/WALs/9d96596e55a9,42339,1731971712494, archiveDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/oldWALs, maxLogs=10 2024-11-18T23:15:15,077 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/WALs/9d96596e55a9,42339,1731971712494/9d96596e55a9%2C42339%2C1731971712494.1731971715060, exclude list is [], retry=0 2024-11-18T23:15:15,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43067,DS-2758df84-89f8-46db-af55-fa2cd38e60fe,DISK] 2024-11-18T23:15:15,098 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-18T23:15:15,141 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/WALs/9d96596e55a9,42339,1731971712494/9d96596e55a9%2C42339%2C1731971712494.1731971715060 2024-11-18T23:15:15,142 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39723:39723)] 2024-11-18T23:15:15,143 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:15:15,143 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:15,148 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,149 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T23:15:15,221 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:15,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:15,224 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T23:15:15,227 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:15,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:15,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T23:15:15,232 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:15,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:15,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T23:15:15,237 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:15,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:15,239 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,249 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,250 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,255 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,256 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,261 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T23:15:15,267 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T23:15:15,274 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:15:15,275 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69623394, jitterRate=0.03746941685676575}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T23:15:15,285 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731971715166Initializing all the Stores at 1731971715168 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731971715169 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971715169Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971715170 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971715170Cleaning up temporary data from old regions at 1731971715257 (+87 ms)Region opened successfully at 1731971715285 (+28 ms) 2024-11-18T23:15:15,286 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T23:15:15,332 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799a24fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d96596e55a9/172.17.0.2:0 2024-11-18T23:15:15,368 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T23:15:15,379 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T23:15:15,379 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T23:15:15,382 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T23:15:15,384 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-18T23:15:15,389 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-18T23:15:15,389 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T23:15:15,425 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T23:15:15,439 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T23:15:15,512 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T23:15:15,515 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T23:15:15,517 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T23:15:15,533 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T23:15:15,536 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T23:15:15,540 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T23:15:15,554 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T23:15:15,556 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T23:15:15,575 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T23:15:15,593 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T23:15:15,596 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T23:15:15,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:15:15,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T23:15:15,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:15,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:15,620 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9d96596e55a9,42339,1731971712494, sessionid=0x101503afdd90000, setting cluster-up flag (Was=false) 2024-11-18T23:15:15,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:15,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:15,713 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T23:15:15,715 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9d96596e55a9,42339,1731971712494 2024-11-18T23:15:15,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:15,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:15,818 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T23:15:15,820 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9d96596e55a9,42339,1731971712494 2024-11-18T23:15:15,826 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T23:15:15,876 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(746): ClusterId : 312958eb-d8b4-4384-a210-556b66aba6db 2024-11-18T23:15:15,879 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T23:15:15,894 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T23:15:15,895 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T23:15:15,904 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T23:15:15,904 DEBUG [RS:0;9d96596e55a9:32899 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ab696c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9d96596e55a9/172.17.0.2:0 2024-11-18T23:15:15,928 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9d96596e55a9:32899 2024-11-18T23:15:15,932 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T23:15:15,932 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T23:15:15,932 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T23:15:15,939 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,42339,1731971712494 with port=32899, startcode=1731971713394 2024-11-18T23:15:15,954 DEBUG [RS:0;9d96596e55a9:32899 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T23:15:15,954 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T23:15:15,974 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T23:15:15,984 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T23:15:15,991 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9d96596e55a9,42339,1731971712494 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T23:15:16,001 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:15:16,002 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:15:16,003 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:15:16,003 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9d96596e55a9:0, corePoolSize=5, maxPoolSize=5 2024-11-18T23:15:16,003 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9d96596e55a9:0, corePoolSize=10, maxPoolSize=10 2024-11-18T23:15:16,003 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,004 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9d96596e55a9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:15:16,004 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,031 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51939, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T23:15:16,031 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731971746031 2024-11-18T23:15:16,034 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T23:15:16,035 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:15:16,035 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T23:15:16,036 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T23:15:16,040 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42339 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:16,042 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T23:15:16,042 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T23:15:16,044 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:16,044 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T23:15:16,050 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T23:15:16,050 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T23:15:16,064 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,074 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T23:15:16,075 WARN [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T23:15:16,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741831_1007 (size=1321) 2024-11-18T23:15:16,081 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T23:15:16,083 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T23:15:16,084 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T23:15:16,086 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T23:15:16,087 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T23:15:16,089 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.large.0-1731971716089,5,FailOnTimeoutGroup] 2024-11-18T23:15:16,092 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.small.0-1731971716090,5,FailOnTimeoutGroup] 2024-11-18T23:15:16,092 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,093 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T23:15:16,094 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,095 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,176 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(2659): reportForDuty to master=9d96596e55a9,42339,1731971712494 with port=32899, startcode=1731971713394 2024-11-18T23:15:16,179 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42339 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9d96596e55a9,32899,1731971713394 2024-11-18T23:15:16,182 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42339 {}] master.ServerManager(517): Registering regionserver=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:16,190 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:15:16,190 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39239 2024-11-18T23:15:16,190 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T23:15:16,207 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:15:16,208 DEBUG [RS:0;9d96596e55a9:32899 {}] zookeeper.ZKUtil(111): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9d96596e55a9,32899,1731971713394 2024-11-18T23:15:16,208 WARN [RS:0;9d96596e55a9:32899 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T23:15:16,208 INFO [RS:0;9d96596e55a9:32899 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:15:16,209 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394 2024-11-18T23:15:16,211 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9d96596e55a9,32899,1731971713394] 2024-11-18T23:15:16,245 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T23:15:16,267 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T23:15:16,273 INFO [RS:0;9d96596e55a9:32899 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T23:15:16,274 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,275 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T23:15:16,281 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T23:15:16,283 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,284 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,284 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,284 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,284 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,284 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,285 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9d96596e55a9:0, corePoolSize=2, maxPoolSize=2 2024-11-18T23:15:16,285 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,285 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,285 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,285 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,286 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,286 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9d96596e55a9:0, corePoolSize=1, maxPoolSize=1 2024-11-18T23:15:16,286 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:15:16,286 DEBUG [RS:0;9d96596e55a9:32899 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0, corePoolSize=3, maxPoolSize=3 2024-11-18T23:15:16,287 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,287 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,288 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,288 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,288 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,288 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,32899,1731971713394-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:15:16,311 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T23:15:16,313 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,32899,1731971713394-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,314 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,314 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.Replication(171): 9d96596e55a9,32899,1731971713394 started 2024-11-18T23:15:16,334 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:16,334 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1482): Serving as 9d96596e55a9,32899,1731971713394, RpcServer on 9d96596e55a9/172.17.0.2:32899, sessionid=0x101503afdd90001 2024-11-18T23:15:16,335 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T23:15:16,335 DEBUG [RS:0;9d96596e55a9:32899 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9d96596e55a9,32899,1731971713394 2024-11-18T23:15:16,335 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,32899,1731971713394' 2024-11-18T23:15:16,335 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T23:15:16,336 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T23:15:16,337 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T23:15:16,337 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T23:15:16,338 DEBUG [RS:0;9d96596e55a9:32899 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9d96596e55a9,32899,1731971713394 2024-11-18T23:15:16,338 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9d96596e55a9,32899,1731971713394' 2024-11-18T23:15:16,338 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T23:15:16,339 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T23:15:16,339 DEBUG [RS:0;9d96596e55a9:32899 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T23:15:16,340 INFO [RS:0;9d96596e55a9:32899 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T23:15:16,340 INFO [RS:0;9d96596e55a9:32899 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T23:15:16,444 INFO [RS:0;9d96596e55a9:32899 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T23:15:16,447 INFO [RS:0;9d96596e55a9:32899 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C32899%2C1731971713394, suffix=, logDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394, archiveDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/oldWALs, maxLogs=32 2024-11-18T23:15:16,465 DEBUG [RS:0;9d96596e55a9:32899 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394/9d96596e55a9%2C32899%2C1731971713394.1731971716450, exclude list is [], retry=0 2024-11-18T23:15:16,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43067,DS-2758df84-89f8-46db-af55-fa2cd38e60fe,DISK] 2024-11-18T23:15:16,473 INFO [RS:0;9d96596e55a9:32899 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394/9d96596e55a9%2C32899%2C1731971713394.1731971716450 2024-11-18T23:15:16,474 DEBUG [RS:0;9d96596e55a9:32899 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39723:39723)] 2024-11-18T23:15:16,481 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T23:15:16,481 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:15:16,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741833_1009 (size=32) 2024-11-18T23:15:16,893 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:16,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T23:15:16,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:15:16,899 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:16,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:16,900 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T23:15:16,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:15:16,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:16,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:16,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T23:15:16,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:15:16,909 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:16,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:16,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T23:15:16,914 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:15:16,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:16,917 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:16,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:15:16,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740 2024-11-18T23:15:16,920 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740 2024-11-18T23:15:16,922 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:15:16,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:15:16,924 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:15:16,926 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:15:16,930 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:15:16,931 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64686386, jitterRate=-0.03609773516654968}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:15:16,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731971716893Initializing all the Stores at 1731971716895 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731971716895Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731971716895Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971716895Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731971716895Cleaning up temporary data from old regions at 1731971716923 (+28 ms)Region opened successfully at 1731971716934 (+11 ms) 2024-11-18T23:15:16,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:15:16,934 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:15:16,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:15:16,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:15:16,934 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:15:16,936 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:15:16,936 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731971716934Disabling compacts and flushes for region at 1731971716934Disabling writes for close at 1731971716934Writing region close event to WAL at 1731971716935 (+1 ms)Closed at 1731971716936 (+1 ms) 2024-11-18T23:15:16,939 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:15:16,939 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T23:15:16,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T23:15:16,952 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:15:16,955 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T23:15:17,108 DEBUG [9d96596e55a9:42339 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T23:15:17,120 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:17,129 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9d96596e55a9,32899,1731971713394, state=OPENING 2024-11-18T23:15:17,154 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T23:15:17,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:17,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:15:17,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:15:17,166 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:15:17,167 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T23:15:17,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:15:17,341 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T23:15:17,344 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60741, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T23:15:17,356 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T23:15:17,356 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T23:15:17,357 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T23:15:17,360 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9d96596e55a9%2C32899%2C1731971713394.meta, suffix=.meta, logDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394, archiveDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/oldWALs, maxLogs=32 2024-11-18T23:15:17,377 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394/9d96596e55a9%2C32899%2C1731971713394.meta.1731971717362.meta, exclude list is [], retry=0 2024-11-18T23:15:17,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43067,DS-2758df84-89f8-46db-af55-fa2cd38e60fe,DISK] 2024-11-18T23:15:17,385 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394/9d96596e55a9%2C32899%2C1731971713394.meta.1731971717362.meta 2024-11-18T23:15:17,385 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39723:39723)] 2024-11-18T23:15:17,386 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:15:17,388 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T23:15:17,390 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T23:15:17,396 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T23:15:17,402 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T23:15:17,403 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:17,403 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T23:15:17,403 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T23:15:17,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-18T23:15:17,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T23:15:17,409 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:17,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:17,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-18T23:15:17,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T23:15:17,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:17,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:17,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-18T23:15:17,414 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T23:15:17,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:17,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:17,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-18T23:15:17,418 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T23:15:17,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:17,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T23:15:17,419 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T23:15:17,421 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740 2024-11-18T23:15:17,424 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740 2024-11-18T23:15:17,427 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T23:15:17,427 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T23:15:17,428 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:15:17,432 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T23:15:17,434 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67625509, jitterRate=0.007698610424995422}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:15:17,434 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T23:15:17,435 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731971717404Writing region info on filesystem at 1731971717404Initializing all the Stores at 1731971717406 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731971717406Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731971717406Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971717406Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731971717406Cleaning up temporary data from old regions at 1731971717427 (+21 ms)Running coprocessor post-open hooks at 1731971717434 (+7 ms)Region opened successfully at 1731971717435 (+1 ms) 2024-11-18T23:15:17,444 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731971717334 2024-11-18T23:15:17,458 DEBUG [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T23:15:17,459 INFO [RS_OPEN_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T23:15:17,461 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:17,464 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9d96596e55a9,32899,1731971713394, state=OPEN 2024-11-18T23:15:17,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:15:17,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T23:15:17,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:15:17,496 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T23:15:17,496 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:17,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T23:15:17,503 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9d96596e55a9,32899,1731971713394 in 328 msec 2024-11-18T23:15:17,511 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T23:15:17,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 560 msec 2024-11-18T23:15:17,513 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T23:15:17,513 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T23:15:17,534 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:17,536 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:17,559 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:17,562 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59783, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:17,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.7140 sec 2024-11-18T23:15:17,585 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731971717585, completionTime=-1 2024-11-18T23:15:17,588 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-18T23:15:17,588 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-18T23:15:17,615 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-18T23:15:17,615 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731971777615 2024-11-18T23:15:17,615 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731971837615 2024-11-18T23:15:17,615 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-18T23:15:17,617 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,42339,1731971712494-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:17,617 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,42339,1731971712494-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:17,617 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,42339,1731971712494-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:17,619 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9d96596e55a9:42339, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:17,619 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:17,620 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:17,625 DEBUG [master/9d96596e55a9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T23:15:17,658 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.113sec 2024-11-18T23:15:17,659 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T23:15:17,661 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T23:15:17,662 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T23:15:17,662 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T23:15:17,663 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T23:15:17,663 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,42339,1731971712494-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T23:15:17,664 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,42339,1731971712494-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T23:15:17,672 DEBUG [master/9d96596e55a9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T23:15:17,672 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T23:15:17,673 INFO [master/9d96596e55a9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9d96596e55a9,42339,1731971712494-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T23:15:17,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:17,687 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T23:15:17,687 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T23:15:17,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:17,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:17,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301f7ee4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:17,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:17,710 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:17,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:17,713 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:17,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:17,722 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:17,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:17,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9d96596e55a9,42339,1731971712494 2024-11-18T23:15:17,757 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=698, ProcessCount=11, AvailableMemoryMB=5975 2024-11-18T23:15:17,765 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:15:17,771 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,42339,1731971712494 2024-11-18T23:15:17,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b1db67 2024-11-18T23:15:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:15:17,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:15:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:15:17,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:15:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:17,817 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:15:17,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-18T23:15:17,819 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:17,826 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:15:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:15:17,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741835_1011 (size=1061) 2024-11-18T23:15:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:15:18,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:15:18,246 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:15:18,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741836_1012 (size=53) 2024-11-18T23:15:18,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:18,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 14ab88ca9857fdcfce5643ad2ad0a965, disabling compactions & flushes 2024-11-18T23:15:18,259 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. after waiting 0 ms 2024-11-18T23:15:18,259 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,259 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,260 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 14ab88ca9857fdcfce5643ad2ad0a965: Waiting for close lock at 1731971718259Disabling compacts and flushes for region at 1731971718259Disabling writes for close at 1731971718259Writing region close event to WAL at 1731971718259Closed at 1731971718259 2024-11-18T23:15:18,262 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:15:18,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731971718262"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731971718262"}]},"ts":"1731971718262"} 2024-11-18T23:15:18,275 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:15:18,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:15:18,282 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971718278"}]},"ts":"1731971718278"} 2024-11-18T23:15:18,288 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T23:15:18,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, ASSIGN}] 2024-11-18T23:15:18,295 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, ASSIGN 2024-11-18T23:15:18,297 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, ASSIGN; state=OFFLINE, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=false 2024-11-18T23:15:18,449 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=14ab88ca9857fdcfce5643ad2ad0a965, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:18,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, ASSIGN because future has completed 2024-11-18T23:15:18,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:15:18,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:15:18,617 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,617 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:15:18,618 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,618 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:18,618 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,618 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,620 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,624 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:18,624 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ab88ca9857fdcfce5643ad2ad0a965 columnFamilyName A 2024-11-18T23:15:18,624 DEBUG [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:18,626 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.HStore(327): Store=14ab88ca9857fdcfce5643ad2ad0a965/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:18,626 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,629 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:18,629 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ab88ca9857fdcfce5643ad2ad0a965 columnFamilyName B 2024-11-18T23:15:18,629 DEBUG [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:18,630 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.HStore(327): Store=14ab88ca9857fdcfce5643ad2ad0a965/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:18,631 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,634 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:18,634 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 14ab88ca9857fdcfce5643ad2ad0a965 columnFamilyName C 2024-11-18T23:15:18,634 DEBUG [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:18,635 INFO [StoreOpener-14ab88ca9857fdcfce5643ad2ad0a965-1 {}] regionserver.HStore(327): Store=14ab88ca9857fdcfce5643ad2ad0a965/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:18,636 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,636 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,638 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,638 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,641 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,642 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,643 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:15:18,647 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,655 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:15:18,656 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 14ab88ca9857fdcfce5643ad2ad0a965; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66212658, jitterRate=-0.013354510068893433}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:15:18,656 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:18,657 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 14ab88ca9857fdcfce5643ad2ad0a965: Running coprocessor pre-open hook at 1731971718618Writing region info on filesystem at 1731971718618Initializing all the Stores at 1731971718620 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971718620Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971718620Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971718620Cleaning up temporary data from old regions at 1731971718642 (+22 ms)Running coprocessor post-open hooks at 1731971718656 (+14 ms)Region opened successfully at 1731971718657 (+1 ms) 2024-11-18T23:15:18,661 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., pid=6, masterSystemTime=1731971718611 2024-11-18T23:15:18,665 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,665 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:18,666 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=14ab88ca9857fdcfce5643ad2ad0a965, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:18,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:15:18,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T23:15:18,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 in 218 msec 2024-11-18T23:15:18,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T23:15:18,686 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, ASSIGN in 389 msec 2024-11-18T23:15:18,689 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:15:18,689 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971718689"}]},"ts":"1731971718689"} 2024-11-18T23:15:18,693 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T23:15:18,696 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:15:18,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 889 msec 2024-11-18T23:15:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T23:15:18,970 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:18,974 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:18,974 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:18,974 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:18,976 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20fa5eed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:18,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:18,978 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:18,980 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:18,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a66f173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:18,985 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,005 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:15:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,42339,1731971712494 2024-11-18T23:15:19,007 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c52dcb8 2024-11-18T23:15:19,010 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:15:19,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:15:19,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,025 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,034 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,038 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fc4f8ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,040 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,041 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,043 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,047 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a084b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,050 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f00f692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,050 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,050 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,052 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0b2729, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,055 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c42c597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388c382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,060 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,060 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,061 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4a2a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,064 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@738bed33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,067 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff89310, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,067 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,067 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,069 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2521eed1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,073 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83804f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,077 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,077 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,079 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f83cf40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,082 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@605ecfde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,092 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2019371a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,092 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,093 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,094 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,094 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b12d099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,096 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,098 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,099 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492f9788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36df70c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,105 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:19,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:19,106 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@684b04c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:19,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27d87de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:19,114 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,114 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,117 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,117 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,117 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,123 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:19,127 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,128 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,131 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,131 DEBUG [Thread-149 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,132 DEBUG [Thread-149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,133 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-18T23:15:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T23:15:19,138 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,140 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,141 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,143 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,145 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:19,146 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,146 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:19,147 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36812, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:19,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:15:19,164 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:19,165 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:19,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:15:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T23:15:19,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:19,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:19,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:19,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:19,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:19,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:19,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T23:15:19,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:19,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:19,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:19,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/9e5df1229d534a119e242f67e54bed34 is 50, key is test_row_0/A:col10/1731971719201/Put/seqid=0 2024-11-18T23:15:19,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971779367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971779383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,440 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971719348, max timestamp=1731971719402Num uniques 859; 2024-11-18T23:15:19,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:19,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971719348, max timestamp=1731971719404Num uniques 859; 2024-11-18T23:15:19,449 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:19,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T23:15:19,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741837_1013 (size=12001) 2024-11-18T23:15:19,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971719348, max timestamp=1731971719402Num uniques 859; 2024-11-18T23:15:19,485 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:15:19,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/9e5df1229d534a119e242f67e54bed34 2024-11-18T23:15:19,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971779548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971779549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971779548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T23:15:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971779555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:19,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971779562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:19,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:19,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/7f9e10e037c9435a9f88f53043ae9688 is 50, key is test_row_0/B:col10/1731971719201/Put/seqid=0 2024-11-18T23:15:19,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741838_1014 (size=12001) 2024-11-18T23:15:19,666 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/7f9e10e037c9435a9f88f53043ae9688 2024-11-18T23:15:19,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971779668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971779669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971779679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971779688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/474a0e5bca464b9b963500855a3cc2e7 is 50, key is test_row_0/C:col10/1731971719201/Put/seqid=0 2024-11-18T23:15:19,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T23:15:19,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:19,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:19,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:19,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:19,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741839_1015 (size=12001) 2024-11-18T23:15:19,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/474a0e5bca464b9b963500855a3cc2e7 2024-11-18T23:15:19,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T23:15:19,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971779769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:19,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:19,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:19,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/9e5df1229d534a119e242f67e54bed34 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e5df1229d534a119e242f67e54bed34 2024-11-18T23:15:19,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e5df1229d534a119e242f67e54bed34, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T23:15:19,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/7f9e10e037c9435a9f88f53043ae9688 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/7f9e10e037c9435a9f88f53043ae9688 2024-11-18T23:15:19,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/7f9e10e037c9435a9f88f53043ae9688, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T23:15:19,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/474a0e5bca464b9b963500855a3cc2e7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/474a0e5bca464b9b963500855a3cc2e7 2024-11-18T23:15:19,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/474a0e5bca464b9b963500855a3cc2e7, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T23:15:19,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.37 KB/123255 for 14ab88ca9857fdcfce5643ad2ad0a965 in 635ms, sequenceid=13, compaction requested=false 2024-11-18T23:15:19,872 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T23:15:19,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:19,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-18T23:15:19,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:19,913 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=120.37 KB heapSize=344.23 KB 2024-11-18T23:15:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:19,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:19,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d0ae6dfc3f03411cb11317b93e9ca4ea is 50, key is test_row_0/A:col10/1731971719556/Put/seqid=0 2024-11-18T23:15:20,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741840_1016 (size=12001) 2024-11-18T23:15:20,009 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d0ae6dfc3f03411cb11317b93e9ca4ea 2024-11-18T23:15:20,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971779986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971779992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971780031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/9cb813bfebb54f65832b61820e5606c6 is 50, key is test_row_0/B:col10/1731971719556/Put/seqid=0 2024-11-18T23:15:20,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971780035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971780088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741841_1017 (size=12001) 2024-11-18T23:15:20,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971780138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971780148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971780158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971780158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,176 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T23:15:20,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971780358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971780369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971780388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971780388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,495 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/9cb813bfebb54f65832b61820e5606c6 2024-11-18T23:15:20,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/737c1d2915324a30977c79549e031d1b is 50, key is test_row_0/C:col10/1731971719556/Put/seqid=0 2024-11-18T23:15:20,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741842_1018 (size=12001) 2024-11-18T23:15:20,583 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/737c1d2915324a30977c79549e031d1b 2024-11-18T23:15:20,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d0ae6dfc3f03411cb11317b93e9ca4ea as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d0ae6dfc3f03411cb11317b93e9ca4ea 2024-11-18T23:15:20,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971780598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,616 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d0ae6dfc3f03411cb11317b93e9ca4ea, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T23:15:20,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/9cb813bfebb54f65832b61820e5606c6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/9cb813bfebb54f65832b61820e5606c6 2024-11-18T23:15:20,644 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/9cb813bfebb54f65832b61820e5606c6, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T23:15:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/737c1d2915324a30977c79549e031d1b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/737c1d2915324a30977c79549e031d1b 2024-11-18T23:15:20,669 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/737c1d2915324a30977c79549e031d1b, entries=150, sequenceid=48, filesize=11.7 K 2024-11-18T23:15:20,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~120.37 KB/123255, heapSize ~344.18 KB/352440, currentSize=60.38 KB/61830 for 14ab88ca9857fdcfce5643ad2ad0a965 in 757ms, sequenceid=48, compaction requested=false 2024-11-18T23:15:20,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:20,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:20,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-18T23:15:20,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-18T23:15:20,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:15:20,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:20,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:20,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:20,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:20,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:20,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:20,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:20,694 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T23:15:20,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5090 sec 2024-11-18T23:15:20,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/019c3687352f427894677fe7c52bf1d6 is 50, key is test_row_0/A:col10/1731971719974/Put/seqid=0 2024-11-18T23:15:20,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5720 sec 2024-11-18T23:15:20,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741843_1019 (size=12001) 2024-11-18T23:15:20,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/019c3687352f427894677fe7c52bf1d6 2024-11-18T23:15:20,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/3187ce08264e4028bb4d1e81f442acee is 50, key is test_row_0/B:col10/1731971719974/Put/seqid=0 2024-11-18T23:15:20,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741844_1020 (size=12001) 2024-11-18T23:15:20,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/3187ce08264e4028bb4d1e81f442acee 2024-11-18T23:15:20,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/342b74f301c041a59fcbbf63a72151e8 is 50, key is test_row_0/C:col10/1731971719974/Put/seqid=0 2024-11-18T23:15:20,839 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971720788, max timestamp=1731971720830Num uniques 859; 2024-11-18T23:15:20,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:15:20,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971720788, max timestamp=1731971720830Num uniques 859; 2024-11-18T23:15:20,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:15:20,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971720788, max timestamp=1731971720830Num uniques 859; 2024-11-18T23:15:20,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:15:20,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741845_1021 (size=12001) 2024-11-18T23:15:20,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/342b74f301c041a59fcbbf63a72151e8 2024-11-18T23:15:20,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/019c3687352f427894677fe7c52bf1d6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/019c3687352f427894677fe7c52bf1d6 2024-11-18T23:15:20,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/019c3687352f427894677fe7c52bf1d6, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:15:20,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/3187ce08264e4028bb4d1e81f442acee as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3187ce08264e4028bb4d1e81f442acee 2024-11-18T23:15:20,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3187ce08264e4028bb4d1e81f442acee, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:15:20,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/342b74f301c041a59fcbbf63a72151e8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/342b74f301c041a59fcbbf63a72151e8 2024-11-18T23:15:20,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971780929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/342b74f301c041a59fcbbf63a72151e8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:15:20,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971780939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:20,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:20,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=120.76 KB/123660 for 14ab88ca9857fdcfce5643ad2ad0a965 in 267ms, sequenceid=61, compaction requested=true 2024-11-18T23:15:20,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:20,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:20,965 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:20,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:20,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:20,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:20,965 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:20,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:20,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:20,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-18T23:15:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:20,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:20,971 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:20,973 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:20,974 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:20,974 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:20,974 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:20,974 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:20,974 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/7f9e10e037c9435a9f88f53043ae9688, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/9cb813bfebb54f65832b61820e5606c6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3187ce08264e4028bb4d1e81f442acee] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=35.2 K 2024-11-18T23:15:20,974 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e5df1229d534a119e242f67e54bed34, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d0ae6dfc3f03411cb11317b93e9ca4ea, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/019c3687352f427894677fe7c52bf1d6] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=35.2 K 2024-11-18T23:15:20,976 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5df1229d534a119e242f67e54bed34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731971719201 2024-11-18T23:15:20,976 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f9e10e037c9435a9f88f53043ae9688, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731971719201 2024-11-18T23:15:20,977 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ae6dfc3f03411cb11317b93e9ca4ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731971719540 2024-11-18T23:15:20,978 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb813bfebb54f65832b61820e5606c6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731971719540 2024-11-18T23:15:20,978 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 019c3687352f427894677fe7c52bf1d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971719974 2024-11-18T23:15:20,978 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3187ce08264e4028bb4d1e81f442acee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971719974 2024-11-18T23:15:20,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/0b2369b1b21e474c8fe17be7f4994220 is 50, key is test_row_0/A:col10/1731971720936/Put/seqid=0 2024-11-18T23:15:20,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741846_1022 (size=16681) 2024-11-18T23:15:20,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/0b2369b1b21e474c8fe17be7f4994220 2024-11-18T23:15:21,041 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:21,045 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/c9ec51786f454dbea3ed17f36ebf3eaa is 50, key is test_row_0/B:col10/1731971719974/Put/seqid=0 2024-11-18T23:15:21,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971781027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,053 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#11 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:21,054 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/9e4bb35047d8474eaf4d94c1ae228569 is 50, key is test_row_0/A:col10/1731971719974/Put/seqid=0 2024-11-18T23:15:21,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ab9e012047e541e09172c8364978832d is 50, key is test_row_0/B:col10/1731971720936/Put/seqid=0 2024-11-18T23:15:21,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971781045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971781048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,069 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971781058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741847_1023 (size=12104) 2024-11-18T23:15:21,107 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/c9ec51786f454dbea3ed17f36ebf3eaa as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/c9ec51786f454dbea3ed17f36ebf3eaa 2024-11-18T23:15:21,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741848_1024 (size=12104) 2024-11-18T23:15:21,141 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into c9ec51786f454dbea3ed17f36ebf3eaa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:21,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:21,144 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971720965; duration=0sec 2024-11-18T23:15:21,145 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:21,145 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:21,146 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:21,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741849_1025 (size=12001) 2024-11-18T23:15:21,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ab9e012047e541e09172c8364978832d 2024-11-18T23:15:21,156 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:21,157 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:21,157 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,158 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/474a0e5bca464b9b963500855a3cc2e7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/737c1d2915324a30977c79549e031d1b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/342b74f301c041a59fcbbf63a72151e8] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=35.2 K 2024-11-18T23:15:21,160 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 474a0e5bca464b9b963500855a3cc2e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731971719201 2024-11-18T23:15:21,161 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 737c1d2915324a30977c79549e031d1b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731971719540 2024-11-18T23:15:21,162 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 342b74f301c041a59fcbbf63a72151e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971719974 2024-11-18T23:15:21,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971781158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971781177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/d19ef0ba41204317bddee5f7b95d3c1a is 50, key is test_row_0/C:col10/1731971720936/Put/seqid=0 2024-11-18T23:15:21,232 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#14 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:21,233 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/dad670b36c9e4461b390711a9e2043ac is 50, key is test_row_0/C:col10/1731971719974/Put/seqid=0 2024-11-18T23:15:21,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741850_1026 (size=12001) 2024-11-18T23:15:21,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-18T23:15:21,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,289 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:21,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971781277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-18T23:15:21,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971781288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T23:15:21,301 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:21,303 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:21,304 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:21,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741851_1027 (size=12104) 2024-11-18T23:15:21,327 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/dad670b36c9e4461b390711a9e2043ac as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/dad670b36c9e4461b390711a9e2043ac 2024-11-18T23:15:21,343 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into dad670b36c9e4461b390711a9e2043ac(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:21,343 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:21,344 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971720965; duration=0sec 2024-11-18T23:15:21,344 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:21,344 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:21,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971781387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971781398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T23:15:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,413 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T23:15:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,568 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/9e4bb35047d8474eaf4d94c1ae228569 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e4bb35047d8474eaf4d94c1ae228569 2024-11-18T23:15:21,591 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into 9e4bb35047d8474eaf4d94c1ae228569(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:21,591 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:21,591 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971720956; duration=0sec 2024-11-18T23:15:21,591 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:21,591 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:21,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971781601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971781607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T23:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:21,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T23:15:21,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971781628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,636 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 14ab88ca9857fdcfce5643ad2ad0a965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,665 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/d19ef0ba41204317bddee5f7b95d3c1a 2024-11-18T23:15:21,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/0b2369b1b21e474c8fe17be7f4994220 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/0b2369b1b21e474c8fe17be7f4994220 2024-11-18T23:15:21,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/0b2369b1b21e474c8fe17be7f4994220, entries=250, sequenceid=96, filesize=16.3 K 2024-11-18T23:15:21,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ab9e012047e541e09172c8364978832d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ab9e012047e541e09172c8364978832d 2024-11-18T23:15:21,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971781717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971781717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:21,726 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:21,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ab9e012047e541e09172c8364978832d, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T23:15:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/d19ef0ba41204317bddee5f7b95d3c1a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d19ef0ba41204317bddee5f7b95d3c1a 2024-11-18T23:15:21,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d19ef0ba41204317bddee5f7b95d3c1a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T23:15:21,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T23:15:21,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:21,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 14ab88ca9857fdcfce5643ad2ad0a965 in 812ms, sequenceid=96, compaction requested=false 2024-11-18T23:15:21,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T23:15:21,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-18T23:15:21,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:21,934 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:15:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:21,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/7d57663ac1d9459ab37b0f9ef9d89dd2 is 50, key is test_row_0/A:col10/1731971720985/Put/seqid=0 2024-11-18T23:15:21,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741852_1028 (size=12001) 2024-11-18T23:15:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:22,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:22,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971722185, max timestamp=1731971722221Num uniques 859; 2024-11-18T23:15:22,235 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:22,236 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T23:15:22,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971722185, max timestamp=1731971722221Num uniques 859; 2024-11-18T23:15:22,245 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:22,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971722185, max timestamp=1731971722221Num uniques 859; 2024-11-18T23:15:22,257 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:15:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971782311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971782318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971782319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971782319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,385 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/7d57663ac1d9459ab37b0f9ef9d89dd2 2024-11-18T23:15:22,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e1b540d9aa9642f1b42d24149d5fb793 is 50, key is test_row_0/B:col10/1731971720985/Put/seqid=0 2024-11-18T23:15:22,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971782438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T23:15:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971782438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971782438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971782438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741853_1029 (size=12001) 2024-11-18T23:15:22,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971782661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971782661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971782661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971782663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,689 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,885 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e1b540d9aa9642f1b42d24149d5fb793 2024-11-18T23:15:22,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/e5e3bf6598154603becb0e42be0de2a0 is 50, key is test_row_0/C:col10/1731971720985/Put/seqid=0 2024-11-18T23:15:22,926 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:15:22,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741854_1030 (size=12001) 2024-11-18T23:15:22,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971782989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971782989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:22,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971782992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:23,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:23,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971783002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:23,087 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T23:15:23,087 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T23:15:23,102 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:15:23,102 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T23:15:23,103 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T23:15:23,103 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T23:15:23,109 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T23:15:23,109 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T23:15:23,369 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/e5e3bf6598154603becb0e42be0de2a0 2024-11-18T23:15:23,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/7d57663ac1d9459ab37b0f9ef9d89dd2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/7d57663ac1d9459ab37b0f9ef9d89dd2 2024-11-18T23:15:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T23:15:23,470 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/7d57663ac1d9459ab37b0f9ef9d89dd2, entries=150, sequenceid=109, filesize=11.7 K 2024-11-18T23:15:23,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e1b540d9aa9642f1b42d24149d5fb793 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e1b540d9aa9642f1b42d24149d5fb793 2024-11-18T23:15:23,529 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e1b540d9aa9642f1b42d24149d5fb793, entries=150, sequenceid=109, filesize=11.7 K 2024-11-18T23:15:23,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/e5e3bf6598154603becb0e42be0de2a0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/e5e3bf6598154603becb0e42be0de2a0 2024-11-18T23:15:23,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971783520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:23,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:23,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971783522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:23,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971783523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:23,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:23,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971783532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:23,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:23,587 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/e5e3bf6598154603becb0e42be0de2a0, entries=150, sequenceid=109, filesize=11.7 K 2024-11-18T23:15:23,600 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1666ms, sequenceid=109, compaction requested=true 2024-11-18T23:15:23,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:23,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:23,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-18T23:15:23,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-18T23:15:23,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-18T23:15:23,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3280 sec 2024-11-18T23:15:23,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.3920 sec 2024-11-18T23:15:23,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731971722299, max timestamp=1731971723655Num uniques 1018; 2024-11-18T23:15:23,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T23:15:23,699 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731971722299, max timestamp=1731971723655Num uniques 1018; 2024-11-18T23:15:23,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T23:15:23,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=64.46 KB heapSize=177.88 KB 2024-11-18T23:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:23,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:23,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:23,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:23,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:15:23,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:23,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/2ad5ab0addd94697ab85982ba3a41afd is 50, key is test_row_0/A:col10/1731971722314/Put/seqid=0 2024-11-18T23:15:23,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741855_1031 (size=14541) 2024-11-18T23:15:23,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/2ad5ab0addd94697ab85982ba3a41afd 2024-11-18T23:15:23,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/5736c244bd2f40f39e3d7561f1463bf1 is 50, key is test_row_0/B:col10/1731971722314/Put/seqid=0 2024-11-18T23:15:23,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741856_1032 (size=12151) 2024-11-18T23:15:24,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971784109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971784228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/5736c244bd2f40f39e3d7561f1463bf1 2024-11-18T23:15:24,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/3c2c6540d04f4068bcd68c91b284feb2 is 50, key is test_row_0/C:col10/1731971722314/Put/seqid=0 2024-11-18T23:15:24,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741857_1033 (size=12151) 2024-11-18T23:15:24,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/3c2c6540d04f4068bcd68c91b284feb2 2024-11-18T23:15:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971784448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/2ad5ab0addd94697ab85982ba3a41afd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2ad5ab0addd94697ab85982ba3a41afd 2024-11-18T23:15:24,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2ad5ab0addd94697ab85982ba3a41afd, entries=200, sequenceid=147, filesize=14.2 K 2024-11-18T23:15:24,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/5736c244bd2f40f39e3d7561f1463bf1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5736c244bd2f40f39e3d7561f1463bf1 2024-11-18T23:15:24,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5736c244bd2f40f39e3d7561f1463bf1, entries=150, sequenceid=147, filesize=11.9 K 2024-11-18T23:15:24,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/3c2c6540d04f4068bcd68c91b284feb2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3c2c6540d04f4068bcd68c91b284feb2 2024-11-18T23:15:24,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3c2c6540d04f4068bcd68c91b284feb2, entries=150, sequenceid=147, filesize=11.9 K 2024-11-18T23:15:24,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.46 KB/66005, heapSize ~177.84 KB/182104, currentSize=114.05 KB/116790 for 14ab88ca9857fdcfce5643ad2ad0a965 in 873ms, sequenceid=147, compaction requested=true 2024-11-18T23:15:24,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T23:15:24,574 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:24,575 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:24,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971723961, max timestamp=1731971724572Num uniques 859; 2024-11-18T23:15:24,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:24,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971723961, max timestamp=1731971724572Num uniques 859; 2024-11-18T23:15:24,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:24,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971723961, max timestamp=1731971724578Num uniques 859; 2024-11-18T23:15:24,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-18T23:15:24,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:24,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:24,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:24,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:24,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:24,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:24,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:24,586 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55327 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:24,586 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:24,586 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:24,587 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:24,587 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:24,587 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:24,587 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/dad670b36c9e4461b390711a9e2043ac, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d19ef0ba41204317bddee5f7b95d3c1a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/e5e3bf6598154603becb0e42be0de2a0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3c2c6540d04f4068bcd68c91b284feb2] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=47.1 K 2024-11-18T23:15:24,587 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e4bb35047d8474eaf4d94c1ae228569, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/0b2369b1b21e474c8fe17be7f4994220, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/7d57663ac1d9459ab37b0f9ef9d89dd2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2ad5ab0addd94697ab85982ba3a41afd] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=54.0 K 2024-11-18T23:15:24,590 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting dad670b36c9e4461b390711a9e2043ac, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971719974 2024-11-18T23:15:24,590 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e4bb35047d8474eaf4d94c1ae228569, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971719974 2024-11-18T23:15:24,592 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d19ef0ba41204317bddee5f7b95d3c1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971720936 2024-11-18T23:15:24,592 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b2369b1b21e474c8fe17be7f4994220, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971720922 2024-11-18T23:15:24,599 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d57663ac1d9459ab37b0f9ef9d89dd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731971720968 2024-11-18T23:15:24,599 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e5e3bf6598154603becb0e42be0de2a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731971720968 2024-11-18T23:15:24,601 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c2c6540d04f4068bcd68c91b284feb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731971722299 2024-11-18T23:15:24,601 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ad5ab0addd94697ab85982ba3a41afd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731971722299 2024-11-18T23:15:24,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/40eeca82e5a04924a5c2e9fa9da85f49 is 50, key is test_row_0/A:col10/1731971724061/Put/seqid=0 2024-11-18T23:15:24,641 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:24,642 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/96f04931c031483a86c8429cf344271c is 50, key is test_row_0/A:col10/1731971722314/Put/seqid=0 2024-11-18T23:15:24,647 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#22 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:24,648 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/06ee867e65f04749943c1f2f21542451 is 50, key is test_row_0/C:col10/1731971722314/Put/seqid=0 2024-11-18T23:15:24,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741858_1034 (size=12151) 2024-11-18T23:15:24,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/40eeca82e5a04924a5c2e9fa9da85f49 2024-11-18T23:15:24,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ccd258b186024b94a5b9f2db245241ad is 50, key is test_row_0/B:col10/1731971724061/Put/seqid=0 2024-11-18T23:15:24,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741859_1035 (size=12391) 2024-11-18T23:15:24,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741860_1036 (size=12391) 2024-11-18T23:15:24,748 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/96f04931c031483a86c8429cf344271c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/96f04931c031483a86c8429cf344271c 2024-11-18T23:15:24,749 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/06ee867e65f04749943c1f2f21542451 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/06ee867e65f04749943c1f2f21542451 2024-11-18T23:15:24,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731971724699, max timestamp=1731971724753Num uniques 859; 2024-11-18T23:15:24,759 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T23:15:24,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731971724699, max timestamp=1731971724757Num uniques 859; 2024-11-18T23:15:24,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731971724699, max timestamp=1731971724757Num uniques 859; 2024-11-18T23:15:24,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T23:15:24,763 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T23:15:24,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741861_1037 (size=12151) 2024-11-18T23:15:24,779 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into 96f04931c031483a86c8429cf344271c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:24,779 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:24,779 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=12, startTime=1731971724574; duration=0sec 2024-11-18T23:15:24,779 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:24,779 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:24,779 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:24,789 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:24,789 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:24,789 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:24,791 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/c9ec51786f454dbea3ed17f36ebf3eaa, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ab9e012047e541e09172c8364978832d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e1b540d9aa9642f1b42d24149d5fb793, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5736c244bd2f40f39e3d7561f1463bf1] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=47.1 K 2024-11-18T23:15:24,792 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9ec51786f454dbea3ed17f36ebf3eaa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971719974 2024-11-18T23:15:24,793 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into 06ee867e65f04749943c1f2f21542451(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:24,793 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:24,793 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=12, startTime=1731971724574; duration=0sec 2024-11-18T23:15:24,793 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:24,793 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:24,794 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab9e012047e541e09172c8364978832d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971720936 2024-11-18T23:15:24,795 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1b540d9aa9642f1b42d24149d5fb793, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731971720968 2024-11-18T23:15:24,796 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5736c244bd2f40f39e3d7561f1463bf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731971722299 2024-11-18T23:15:24,838 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#25 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:24,839 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/f3f3ee3127e04832817be9d2e5d28e48 is 50, key is test_row_0/B:col10/1731971722314/Put/seqid=0 2024-11-18T23:15:24,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971784834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971784848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971784851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971784844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971784847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741862_1038 (size=12391) 2024-11-18T23:15:24,910 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/f3f3ee3127e04832817be9d2e5d28e48 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f3ee3127e04832817be9d2e5d28e48 2024-11-18T23:15:24,929 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into f3f3ee3127e04832817be9d2e5d28e48(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:24,929 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:24,929 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=12, startTime=1731971724574; duration=0sec 2024-11-18T23:15:24,930 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:24,930 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:24,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971784958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971784971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971784971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:24,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971784981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:24,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971784997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ccd258b186024b94a5b9f2db245241ad 2024-11-18T23:15:25,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971785188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971785202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971785202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971785203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6 is 50, key is test_row_0/C:col10/1731971724061/Put/seqid=0 2024-11-18T23:15:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971785208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,228 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741863_1039 (size=12151) 2024-11-18T23:15:25,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6 2024-11-18T23:15:25,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/40eeca82e5a04924a5c2e9fa9da85f49 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/40eeca82e5a04924a5c2e9fa9da85f49 2024-11-18T23:15:25,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/40eeca82e5a04924a5c2e9fa9da85f49, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T23:15:25,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ccd258b186024b94a5b9f2db245241ad as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ccd258b186024b94a5b9f2db245241ad 2024-11-18T23:15:25,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ccd258b186024b94a5b9f2db245241ad, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T23:15:25,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6 2024-11-18T23:15:25,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6, entries=150, sequenceid=169, filesize=11.9 K 2024-11-18T23:15:25,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 14ab88ca9857fdcfce5643ad2ad0a965 in 742ms, sequenceid=169, compaction requested=false 2024-11-18T23:15:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-18T23:15:25,473 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:25,474 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-18T23:15:25,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T23:15:25,526 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:25,526 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:25,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=161.02 KB heapSize=451.31 KB 2024-11-18T23:15:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:25,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:25,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:15:25,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:25,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:15:25,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:25,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:15:25,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d27f752edf33427aa31c794fda3ffc01 is 50, key is test_row_0/A:col10/1731971725545/Put/seqid=0 2024-11-18T23:15:25,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971785549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971785552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971785561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971785574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971785581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741864_1040 (size=14541) 2024-11-18T23:15:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T23:15:25,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T23:15:25,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:25,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:25,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:25,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:25,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971785698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971785699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971785700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971785698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971785700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T23:15:25,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T23:15:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:25,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:25,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:25,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971785928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971785935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971785935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971785935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:25,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971785947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:25,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d27f752edf33427aa31c794fda3ffc01 2024-11-18T23:15:26,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T23:15:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/6d2ae24f7b9141aebf655271314e199e is 50, key is test_row_0/B:col10/1731971725545/Put/seqid=0 2024-11-18T23:15:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T23:15:26,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741865_1041 (size=12151) 2024-11-18T23:15:26,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/6d2ae24f7b9141aebf655271314e199e 2024-11-18T23:15:26,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T23:15:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971786263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971786264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971786263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971786265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971786268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/d8da56a8f3ab4437a4948d4a00171c32 is 50, key is test_row_0/C:col10/1731971725545/Put/seqid=0 2024-11-18T23:15:26,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741866_1042 (size=12151) 2024-11-18T23:15:26,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/d8da56a8f3ab4437a4948d4a00171c32 2024-11-18T23:15:26,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T23:15:26,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:26,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d27f752edf33427aa31c794fda3ffc01 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d27f752edf33427aa31c794fda3ffc01 2024-11-18T23:15:26,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d27f752edf33427aa31c794fda3ffc01, entries=200, sequenceid=211, filesize=14.2 K 2024-11-18T23:15:26,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/6d2ae24f7b9141aebf655271314e199e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/6d2ae24f7b9141aebf655271314e199e 2024-11-18T23:15:26,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/6d2ae24f7b9141aebf655271314e199e, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T23:15:26,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/d8da56a8f3ab4437a4948d4a00171c32 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d8da56a8f3ab4437a4948d4a00171c32 2024-11-18T23:15:26,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T23:15:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:26,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d8da56a8f3ab4437a4948d4a00171c32, entries=150, sequenceid=211, filesize=11.9 K 2024-11-18T23:15:26,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~471.19 KB/482496, currentSize=20.13 KB/20610 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1041ms, sequenceid=211, compaction requested=true 2024-11-18T23:15:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:26,589 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:26,589 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:26,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:26,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:26,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:26,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:26,596 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:26,600 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39083 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:26,600 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:26,600 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,601 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/96f04931c031483a86c8429cf344271c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/40eeca82e5a04924a5c2e9fa9da85f49, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d27f752edf33427aa31c794fda3ffc01] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=38.2 K 2024-11-18T23:15:26,603 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f04931c031483a86c8429cf344271c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731971722299 2024-11-18T23:15:26,607 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40eeca82e5a04924a5c2e9fa9da85f49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731971724061 2024-11-18T23:15:26,607 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:26,607 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:26,607 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,608 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f3ee3127e04832817be9d2e5d28e48, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ccd258b186024b94a5b9f2db245241ad, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/6d2ae24f7b9141aebf655271314e199e] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=35.8 K 2024-11-18T23:15:26,608 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d27f752edf33427aa31c794fda3ffc01, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731971724844 2024-11-18T23:15:26,609 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f3ee3127e04832817be9d2e5d28e48, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731971722299 2024-11-18T23:15:26,610 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ccd258b186024b94a5b9f2db245241ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731971724061 2024-11-18T23:15:26,610 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d2ae24f7b9141aebf655271314e199e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731971724848 2024-11-18T23:15:26,637 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:26,638 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/6d9dfa8fe2744ea5abd835e3919f6982 is 50, key is test_row_0/A:col10/1731971725545/Put/seqid=0 2024-11-18T23:15:26,655 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:26,656 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e75e12ee529a462296c39d439d1958a3 is 50, key is test_row_0/B:col10/1731971725545/Put/seqid=0 2024-11-18T23:15:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T23:15:26,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741867_1043 (size=12493) 2024-11-18T23:15:26,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741868_1044 (size=12493) 2024-11-18T23:15:26,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-18T23:15:26,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,745 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T23:15:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:26,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:26,772 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e75e12ee529a462296c39d439d1958a3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e75e12ee529a462296c39d439d1958a3 2024-11-18T23:15:26,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/431ae34b26264e34a7e1af9df46b2130 is 50, key is test_row_0/A:col10/1731971725559/Put/seqid=0 2024-11-18T23:15:26,793 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into e75e12ee529a462296c39d439d1958a3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:26,793 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:26,793 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971726590; duration=0sec 2024-11-18T23:15:26,794 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:26,794 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:26,794 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:26,796 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:26,796 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:26,796 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:26,796 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/06ee867e65f04749943c1f2f21542451, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d8da56a8f3ab4437a4948d4a00171c32] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=35.8 K 2024-11-18T23:15:26,802 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 06ee867e65f04749943c1f2f21542451, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731971722299 2024-11-18T23:15:26,803 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6dd9bfe0c34b9eb8078b0ebb8891d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731971724061 2024-11-18T23:15:26,804 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d8da56a8f3ab4437a4948d4a00171c32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731971724848 2024-11-18T23:15:26,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:26,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741869_1045 (size=9757) 2024-11-18T23:15:26,861 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#33 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:26,862 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/349b88ab90de44b08883a45e003e8a32 is 50, key is test_row_0/C:col10/1731971725545/Put/seqid=0 2024-11-18T23:15:26,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971726852, max timestamp=1731971726903Num uniques 859; 2024-11-18T23:15:26,915 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:15:26,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741870_1046 (size=12493) 2024-11-18T23:15:26,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971786965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971786965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971786966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:26,987 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:26,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971786973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971786978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971787099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971787099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971787100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971787109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,122 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,134 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/6d9dfa8fe2744ea5abd835e3919f6982 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/6d9dfa8fe2744ea5abd835e3919f6982 2024-11-18T23:15:27,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971787121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,161 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into 6d9dfa8fe2744ea5abd835e3919f6982(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:27,161 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:27,161 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971726589; duration=0sec 2024-11-18T23:15:27,161 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:27,161 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:27,255 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/431ae34b26264e34a7e1af9df46b2130 2024-11-18T23:15:27,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ce2fe14dbad4469486a6419d88fc677c is 50, key is test_row_0/B:col10/1731971725559/Put/seqid=0 2024-11-18T23:15:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971787327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971787328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,343 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/349b88ab90de44b08883a45e003e8a32 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/349b88ab90de44b08883a45e003e8a32 2024-11-18T23:15:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971787327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971787328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,367 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into 349b88ab90de44b08883a45e003e8a32(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:27,367 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:27,368 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971726590; duration=0sec 2024-11-18T23:15:27,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741871_1047 (size=9757) 2024-11-18T23:15:27,368 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:27,368 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:27,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,378 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ce2fe14dbad4469486a6419d88fc677c 2024-11-18T23:15:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971787348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/4357ef97862843b0a7f8d93a88ce67c0 is 50, key is test_row_0/C:col10/1731971725559/Put/seqid=0 2024-11-18T23:15:27,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741872_1048 (size=9757) 2024-11-18T23:15:27,472 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/4357ef97862843b0a7f8d93a88ce67c0 2024-11-18T23:15:27,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/431ae34b26264e34a7e1af9df46b2130 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/431ae34b26264e34a7e1af9df46b2130 2024-11-18T23:15:27,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/431ae34b26264e34a7e1af9df46b2130, entries=100, sequenceid=217, filesize=9.5 K 2024-11-18T23:15:27,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/ce2fe14dbad4469486a6419d88fc677c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ce2fe14dbad4469486a6419d88fc677c 2024-11-18T23:15:27,565 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ce2fe14dbad4469486a6419d88fc677c, entries=100, sequenceid=217, filesize=9.5 K 2024-11-18T23:15:27,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/4357ef97862843b0a7f8d93a88ce67c0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4357ef97862843b0a7f8d93a88ce67c0 2024-11-18T23:15:27,597 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4357ef97862843b0a7f8d93a88ce67c0, entries=100, sequenceid=217, filesize=9.5 K 2024-11-18T23:15:27,602 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=154.31 KB/158010 for 14ab88ca9857fdcfce5643ad2ad0a965 in 857ms, sequenceid=217, compaction requested=false 2024-11-18T23:15:27,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:27,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:27,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-18T23:15:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-18T23:15:27,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-18T23:15:27,631 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0870 sec 2024-11-18T23:15:27,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.1580 sec 2024-11-18T23:15:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:27,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=161.02 KB heapSize=471.48 KB 2024-11-18T23:15:27,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:27,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:27,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:27,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:27,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:27,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-18T23:15:27,678 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:27,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-18T23:15:27,682 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:27,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971787668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971787668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,685 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:27,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T23:15:27,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971787673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971787686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/07925b3fd2c04428aae0cc95a90514a6 is 50, key is test_row_0/A:col10/1731971726959/Put/seqid=0 2024-11-18T23:15:27,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971787704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741873_1049 (size=14541) 2024-11-18T23:15:27,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/07925b3fd2c04428aae0cc95a90514a6 2024-11-18T23:15:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T23:15:27,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971787799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971787799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:27,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e766fc433245472ab37df163fbfeccd1 is 50, key is test_row_0/B:col10/1731971726959/Put/seqid=0 2024-11-18T23:15:27,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741874_1050 (size=12151) 2024-11-18T23:15:27,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e766fc433245472ab37df163fbfeccd1 2024-11-18T23:15:27,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T23:15:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:27,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:27,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:27,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:27,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/f455ff19147e4627b7a210597680354d is 50, key is test_row_0/C:col10/1731971726959/Put/seqid=0 2024-11-18T23:15:27,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741875_1051 (size=12151) 2024-11-18T23:15:27,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/f455ff19147e4627b7a210597680354d 2024-11-18T23:15:27,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/07925b3fd2c04428aae0cc95a90514a6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/07925b3fd2c04428aae0cc95a90514a6 2024-11-18T23:15:28,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T23:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:28,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:28,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:28,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T23:15:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:28,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971788011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/07925b3fd2c04428aae0cc95a90514a6, entries=200, sequenceid=251, filesize=14.2 K 2024-11-18T23:15:28,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971788017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/e766fc433245472ab37df163fbfeccd1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e766fc433245472ab37df163fbfeccd1 2024-11-18T23:15:28,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e766fc433245472ab37df163fbfeccd1, entries=150, sequenceid=251, filesize=11.9 K 2024-11-18T23:15:28,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/f455ff19147e4627b7a210597680354d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/f455ff19147e4627b7a210597680354d 2024-11-18T23:15:28,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/f455ff19147e4627b7a210597680354d, entries=150, sequenceid=251, filesize=11.9 K 2024-11-18T23:15:28,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~471.44 KB/482752, currentSize=13.42 KB/13740 for 14ab88ca9857fdcfce5643ad2ad0a965 in 449ms, sequenceid=251, compaction requested=true 2024-11-18T23:15:28,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:28,106 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:28,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:28,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:28,107 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:28,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:28,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:28,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:28,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:28,113 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:28,113 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:28,113 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:28,113 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/6d9dfa8fe2744ea5abd835e3919f6982, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/431ae34b26264e34a7e1af9df46b2130, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/07925b3fd2c04428aae0cc95a90514a6] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=35.9 K 2024-11-18T23:15:28,114 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:28,114 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:28,114 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:28,114 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e75e12ee529a462296c39d439d1958a3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ce2fe14dbad4469486a6419d88fc677c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e766fc433245472ab37df163fbfeccd1] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=33.6 K 2024-11-18T23:15:28,124 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e75e12ee529a462296c39d439d1958a3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731971724848 2024-11-18T23:15:28,125 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d9dfa8fe2744ea5abd835e3919f6982, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731971724848 2024-11-18T23:15:28,126 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2fe14dbad4469486a6419d88fc677c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971725558 2024-11-18T23:15:28,126 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 431ae34b26264e34a7e1af9df46b2130, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971725558 2024-11-18T23:15:28,133 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e766fc433245472ab37df163fbfeccd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731971726959 2024-11-18T23:15:28,133 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07925b3fd2c04428aae0cc95a90514a6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731971726959 2024-11-18T23:15:28,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-18T23:15:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:28,172 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-18T23:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:28,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:28,179 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:28,179 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/80c9e5fc87244028aa2af67da4484d32 is 50, key is test_row_0/A:col10/1731971726959/Put/seqid=0 2024-11-18T23:15:28,188 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:28,189 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/eaa3172b04364be7957a14cb5a012b33 is 50, key is test_row_0/B:col10/1731971726959/Put/seqid=0 2024-11-18T23:15:28,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/86a5a2745d0543e6adff0e207f1cb596 is 50, key is test_row_0/A:col10/1731971727668/Put/seqid=0 2024-11-18T23:15:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:28,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741876_1052 (size=12595) 2024-11-18T23:15:28,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741877_1053 (size=12595) 2024-11-18T23:15:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T23:15:28,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741878_1054 (size=9757) 2024-11-18T23:15:28,336 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/86a5a2745d0543e6adff0e207f1cb596 2024-11-18T23:15:28,358 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/80c9e5fc87244028aa2af67da4484d32 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/80c9e5fc87244028aa2af67da4484d32 2024-11-18T23:15:28,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971728321, max timestamp=1731971728349Num uniques 859; 2024-11-18T23:15:28,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:15:28,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971728321, max timestamp=1731971728365Num uniques 859; 2024-11-18T23:15:28,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:15:28,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971728321, max timestamp=1731971728349Num uniques 859; 2024-11-18T23:15:28,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:15:28,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0d7ebb331f2045ec8aea6418099f7965 is 50, key is test_row_0/B:col10/1731971727668/Put/seqid=0 2024-11-18T23:15:28,399 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into 80c9e5fc87244028aa2af67da4484d32(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:28,399 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:28,399 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971728106; duration=0sec 2024-11-18T23:15:28,400 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:28,400 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:28,400 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:28,406 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:28,406 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:28,406 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:28,406 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/349b88ab90de44b08883a45e003e8a32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4357ef97862843b0a7f8d93a88ce67c0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/f455ff19147e4627b7a210597680354d] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=33.6 K 2024-11-18T23:15:28,413 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 349b88ab90de44b08883a45e003e8a32, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731971724848 2024-11-18T23:15:28,415 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4357ef97862843b0a7f8d93a88ce67c0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971725558 2024-11-18T23:15:28,417 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f455ff19147e4627b7a210597680354d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731971726959 2024-11-18T23:15:28,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741879_1055 (size=9757) 2024-11-18T23:15:28,435 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:28,435 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/fd46b45aa4974d289b6618cee25c87c4 is 50, key is test_row_0/C:col10/1731971726959/Put/seqid=0 2024-11-18T23:15:28,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741880_1056 (size=12595) 2024-11-18T23:15:28,473 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/fd46b45aa4974d289b6618cee25c87c4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/fd46b45aa4974d289b6618cee25c87c4 2024-11-18T23:15:28,497 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into fd46b45aa4974d289b6618cee25c87c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:28,497 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:28,497 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971728108; duration=0sec 2024-11-18T23:15:28,498 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:28,498 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:28,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1731971728399, max timestamp=1731971728498Num uniques 1068; 2024-11-18T23:15:28,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1731971728399, max timestamp=1731971728498Num uniques 1068; 2024-11-18T23:15:28,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=26008, suffix heap size=139040,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=568 2024-11-18T23:15:28,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=26008, suffix heap size=139040,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=568 2024-11-18T23:15:28,518 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=568, cellSize=26008, totalHeapSize=59312, min timestamp=1731971728399, max timestamp=1731971728498Num uniques 1009; 2024-11-18T23:15:28,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=26008, suffix heap size=132904,new segment heap size=59312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=568 2024-11-18T23:15:28,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971788615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971788627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971788631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971788640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971788628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,761 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/eaa3172b04364be7957a14cb5a012b33 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/eaa3172b04364be7957a14cb5a012b33 2024-11-18T23:15:28,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971788747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971788758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971788759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971788758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971788768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,791 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into eaa3172b04364be7957a14cb5a012b33(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:28,796 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:28,796 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971728107; duration=0sec 2024-11-18T23:15:28,796 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:28,796 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:28,829 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0d7ebb331f2045ec8aea6418099f7965 2024-11-18T23:15:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T23:15:28,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/4e9f6621b02a4124a77e71751567888d is 50, key is test_row_0/C:col10/1731971727668/Put/seqid=0 2024-11-18T23:15:28,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741881_1057 (size=9757) 2024-11-18T23:15:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971788977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971788979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971788979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971788987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971788987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:29,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971789289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:29,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971789289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:29,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971789289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:29,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971789297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:29,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971789298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:29,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:29,336 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/4e9f6621b02a4124a77e71751567888d 2024-11-18T23:15:29,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/86a5a2745d0543e6adff0e207f1cb596 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/86a5a2745d0543e6adff0e207f1cb596 2024-11-18T23:15:29,421 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/86a5a2745d0543e6adff0e207f1cb596, entries=100, sequenceid=256, filesize=9.5 K 2024-11-18T23:15:29,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0d7ebb331f2045ec8aea6418099f7965 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0d7ebb331f2045ec8aea6418099f7965 2024-11-18T23:15:29,431 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0d7ebb331f2045ec8aea6418099f7965, entries=100, sequenceid=256, filesize=9.5 K 2024-11-18T23:15:29,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/4e9f6621b02a4124a77e71751567888d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4e9f6621b02a4124a77e71751567888d 2024-11-18T23:15:29,450 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4e9f6621b02a4124a77e71751567888d, entries=100, sequenceid=256, filesize=9.5 K 2024-11-18T23:15:29,460 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=187.85 KB/192360 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1288ms, sequenceid=256, compaction requested=false 2024-11-18T23:15:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:29,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-18T23:15:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-18T23:15:29,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-18T23:15:29,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7910 sec 2024-11-18T23:15:29,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.8320 sec 2024-11-18T23:15:29,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731971728605, max timestamp=1731971729816Num uniques 1427; 2024-11-18T23:15:29,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=65352, new segment data size=8102, suffix heap size=176376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1427, new segment cells count=177 2024-11-18T23:15:29,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731971728605, max timestamp=1731971729816Num uniques 1427; 2024-11-18T23:15:29,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=65352, new segment data size=8102, suffix heap size=176376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1427, new segment cells count=177 2024-11-18T23:15:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:29,830 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731971728605, max timestamp=1731971729816Num uniques 1427; 2024-11-18T23:15:29,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=65352, new segment data size=8102, suffix heap size=176376,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1427, new segment cells count=177 2024-11-18T23:15:29,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=26.84 KB heapSize=64.55 KB 2024-11-18T23:15:29,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:29,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-18T23:15:29,851 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:29,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:29,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-18T23:15:29,856 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:29,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/81cfea96921e46c79ad08795a6397255 is 50, key is test_row_0/A:col10/1731971729825/Put/seqid=0 2024-11-18T23:15:29,859 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:29,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:29,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T23:15:29,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731971729864, max timestamp=1731971729924Num uniques 859; 2024-11-18T23:15:29,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-18T23:15:29,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731971729864, max timestamp=1731971729924Num uniques 859; 2024-11-18T23:15:29,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-18T23:15:29,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=609, cellSize=27885, totalHeapSize=63576, min timestamp=1731971729864, max timestamp=1731971729924Num uniques 859; 2024-11-18T23:15:29,939 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=27885, suffix heap size=117064,new segment heap size=63576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=609 2024-11-18T23:15:29,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741882_1058 (size=17181) 2024-11-18T23:15:29,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/81cfea96921e46c79ad08795a6397255 2024-11-18T23:15:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T23:15:30,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/889a4279a14d4afe85710c48a303aad9 is 50, key is test_row_0/B:col10/1731971729825/Put/seqid=0 2024-11-18T23:15:30,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T23:15:30,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,030 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971789997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971789998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971790008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,040 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971790007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971790019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741883_1059 (size=12301) 2024-11-18T23:15:30,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/889a4279a14d4afe85710c48a303aad9 2024-11-18T23:15:30,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/35f7bd1096fb416190ad932a204f6157 is 50, key is test_row_0/C:col10/1731971729825/Put/seqid=0 2024-11-18T23:15:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971790153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971790153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971790154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971790154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971790161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T23:15:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:30,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T23:15:30,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741884_1060 (size=12301) 2024-11-18T23:15:30,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/35f7bd1096fb416190ad932a204f6157 2024-11-18T23:15:30,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/81cfea96921e46c79ad08795a6397255 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/81cfea96921e46c79ad08795a6397255 2024-11-18T23:15:30,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/81cfea96921e46c79ad08795a6397255, entries=250, sequenceid=315, filesize=16.8 K 2024-11-18T23:15:30,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/889a4279a14d4afe85710c48a303aad9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/889a4279a14d4afe85710c48a303aad9 2024-11-18T23:15:30,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T23:15:30,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:30,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:30,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/889a4279a14d4afe85710c48a303aad9, entries=150, sequenceid=315, filesize=12.0 K 2024-11-18T23:15:30,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971790371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/35f7bd1096fb416190ad932a204f6157 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/35f7bd1096fb416190ad932a204f6157 2024-11-18T23:15:30,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971790372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971790372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971790381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971790384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/35f7bd1096fb416190ad932a204f6157, entries=150, sequenceid=315, filesize=12.0 K 2024-11-18T23:15:30,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~84.42 KB/86448, currentSize=167.72 KB/171750 for 14ab88ca9857fdcfce5643ad2ad0a965 in 622ms, sequenceid=315, compaction requested=true 2024-11-18T23:15:30,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:30,457 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:30,462 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:30,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:30,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:30,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:30,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:30,468 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:30,468 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:30,468 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,468 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/80c9e5fc87244028aa2af67da4484d32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/86a5a2745d0543e6adff0e207f1cb596, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/81cfea96921e46c79ad08795a6397255] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=38.6 K 2024-11-18T23:15:30,472 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:30,472 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:30,473 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,473 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/eaa3172b04364be7957a14cb5a012b33, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0d7ebb331f2045ec8aea6418099f7965, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/889a4279a14d4afe85710c48a303aad9] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=33.8 K 2024-11-18T23:15:30,475 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 80c9e5fc87244028aa2af67da4484d32, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731971726959 2024-11-18T23:15:30,477 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa3172b04364be7957a14cb5a012b33, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731971726959 2024-11-18T23:15:30,477 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 86a5a2745d0543e6adff0e207f1cb596, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731971727668 2024-11-18T23:15:30,483 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7ebb331f2045ec8aea6418099f7965, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731971727668 2024-11-18T23:15:30,484 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 81cfea96921e46c79ad08795a6397255, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731971728605 2024-11-18T23:15:30,486 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 889a4279a14d4afe85710c48a303aad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731971728620 2024-11-18T23:15:30,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T23:15:30,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-18T23:15:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:30,521 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=167.72 KB heapSize=442.41 KB 2024-11-18T23:15:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:30,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:30,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:30,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:30,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:30,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:30,530 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:30,534 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/05b7d9bbad1f4ef09193a911e1f87138 is 50, key is test_row_0/B:col10/1731971729825/Put/seqid=0 2024-11-18T23:15:30,544 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#49 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:30,544 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d21039c119c440a987546cfb214849ad is 50, key is test_row_0/A:col10/1731971729825/Put/seqid=0 2024-11-18T23:15:30,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/8c4cf04d97da484d8eafd3b0196f6e44 is 50, key is test_row_0/A:col10/1731971730006/Put/seqid=0 2024-11-18T23:15:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741886_1062 (size=12847) 2024-11-18T23:15:30,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741885_1061 (size=12847) 2024-11-18T23:15:30,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741887_1063 (size=12301) 2024-11-18T23:15:30,612 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/8c4cf04d97da484d8eafd3b0196f6e44 2024-11-18T23:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/bc2f276dfc1741ab91c559905fa4fe05 is 50, key is test_row_0/B:col10/1731971730006/Put/seqid=0 2024-11-18T23:15:30,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:30,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:30,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971790717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971790727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971790727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741888_1064 (size=12301) 2024-11-18T23:15:30,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971790733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971790738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971790847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971790847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971790867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971790868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:30,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:30,878 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T23:15:31,050 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d21039c119c440a987546cfb214849ad as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d21039c119c440a987546cfb214849ad 2024-11-18T23:15:31,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971791068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971791068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,078 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/05b7d9bbad1f4ef09193a911e1f87138 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/05b7d9bbad1f4ef09193a911e1f87138 2024-11-18T23:15:31,090 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into d21039c119c440a987546cfb214849ad(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:31,090 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:31,090 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971730455; duration=0sec 2024-11-18T23:15:31,090 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:31,090 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:31,090 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:31,096 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:31,096 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:31,096 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:31,097 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/fd46b45aa4974d289b6618cee25c87c4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4e9f6621b02a4124a77e71751567888d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/35f7bd1096fb416190ad932a204f6157] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=33.8 K 2024-11-18T23:15:31,100 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into 05b7d9bbad1f4ef09193a911e1f87138(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:31,100 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:31,100 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971730462; duration=0sec 2024-11-18T23:15:31,100 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:31,100 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:31,102 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting fd46b45aa4974d289b6618cee25c87c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731971726959 2024-11-18T23:15:31,104 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e9f6621b02a4124a77e71751567888d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731971727668 2024-11-18T23:15:31,109 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f7bd1096fb416190ad932a204f6157, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731971728620 2024-11-18T23:15:31,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971791091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971791094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,155 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/bc2f276dfc1741ab91c559905fa4fe05 2024-11-18T23:15:31,160 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:31,164 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/5617401f9b5f457c9d6cb7828b7904e5 is 50, key is test_row_0/C:col10/1731971729825/Put/seqid=0 2024-11-18T23:15:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/44d3c9169fba4edeb1bb10b2d92e619d is 50, key is test_row_0/C:col10/1731971730006/Put/seqid=0 2024-11-18T23:15:31,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741889_1065 (size=12847) 2024-11-18T23:15:31,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741890_1066 (size=12301) 2024-11-18T23:15:31,209 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/44d3c9169fba4edeb1bb10b2d92e619d 2024-11-18T23:15:31,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/8c4cf04d97da484d8eafd3b0196f6e44 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/8c4cf04d97da484d8eafd3b0196f6e44 2024-11-18T23:15:31,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971791248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,256 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/8c4cf04d97da484d8eafd3b0196f6e44, entries=150, sequenceid=348, filesize=12.0 K 2024-11-18T23:15:31,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/bc2f276dfc1741ab91c559905fa4fe05 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/bc2f276dfc1741ab91c559905fa4fe05 2024-11-18T23:15:31,295 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/bc2f276dfc1741ab91c559905fa4fe05, entries=150, sequenceid=348, filesize=12.0 K 2024-11-18T23:15:31,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/44d3c9169fba4edeb1bb10b2d92e619d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/44d3c9169fba4edeb1bb10b2d92e619d 2024-11-18T23:15:31,352 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/44d3c9169fba4edeb1bb10b2d92e619d, entries=150, sequenceid=348, filesize=12.0 K 2024-11-18T23:15:31,353 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~442.36 KB/452976, currentSize=26.84 KB/27480 for 14ab88ca9857fdcfce5643ad2ad0a965 in 832ms, sequenceid=348, compaction requested=false 2024-11-18T23:15:31,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:31,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:31,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-18T23:15:31,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-18T23:15:31,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-18T23:15:31,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-18T23:15:31,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.5170 sec 2024-11-18T23:15:31,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:15:31,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:31,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/fde22ec320a740b88b4ba2740b8d5d1a is 50, key is test_row_0/A:col10/1731971731416/Put/seqid=0 2024-11-18T23:15:31,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741891_1067 (size=12301) 2024-11-18T23:15:31,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/fde22ec320a740b88b4ba2740b8d5d1a 2024-11-18T23:15:31,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971731531, max timestamp=1731971731585Num uniques 859; 2024-11-18T23:15:31,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:15:31,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971731531, max timestamp=1731971731585Num uniques 859; 2024-11-18T23:15:31,593 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:15:31,594 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971731531, max timestamp=1731971731585Num uniques 859; 2024-11-18T23:15:31,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:15:31,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/f3f0f084cb1c4923831fa540ca66c1a8 is 50, key is test_row_0/B:col10/1731971731416/Put/seqid=0 2024-11-18T23:15:31,674 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/5617401f9b5f457c9d6cb7828b7904e5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5617401f9b5f457c9d6cb7828b7904e5 2024-11-18T23:15:31,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741892_1068 (size=12301) 2024-11-18T23:15:31,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971791701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971791706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,720 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into 5617401f9b5f457c9d6cb7828b7904e5(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:31,720 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:31,720 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971730463; duration=0sec 2024-11-18T23:15:31,721 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:31,722 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971791707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971791709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971791820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971791828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971791838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971791840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:31,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-18T23:15:32,029 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:32,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-18T23:15:32,044 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T23:15:32,045 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:32,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:32,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971792058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971792058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971792060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971792063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/f3f0f084cb1c4923831fa540ca66c1a8 2024-11-18T23:15:32,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/79f7621c7bbb475996ff48c44b3d8b9d is 50, key is test_row_0/C:col10/1731971731416/Put/seqid=0 2024-11-18T23:15:32,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T23:15:32,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741893_1069 (size=12301) 2024-11-18T23:15:32,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T23:15:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:32,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971792268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,272 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 14ab88ca9857fdcfce5643ad2ad0a965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T23:15:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971792378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971792378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971792378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,384 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971792378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:32,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:32,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T23:15:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T23:15:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:32,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/79f7621c7bbb475996ff48c44b3d8b9d 2024-11-18T23:15:32,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/fde22ec320a740b88b4ba2740b8d5d1a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fde22ec320a740b88b4ba2740b8d5d1a 2024-11-18T23:15:32,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fde22ec320a740b88b4ba2740b8d5d1a, entries=150, sequenceid=361, filesize=12.0 K 2024-11-18T23:15:32,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/f3f0f084cb1c4923831fa540ca66c1a8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f0f084cb1c4923831fa540ca66c1a8 2024-11-18T23:15:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T23:15:32,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f0f084cb1c4923831fa540ca66c1a8, entries=150, sequenceid=361, filesize=12.0 K 2024-11-18T23:15:32,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/79f7621c7bbb475996ff48c44b3d8b9d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/79f7621c7bbb475996ff48c44b3d8b9d 2024-11-18T23:15:32,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/79f7621c7bbb475996ff48c44b3d8b9d, entries=150, sequenceid=361, filesize=12.0 K 2024-11-18T23:15:32,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1272ms, sequenceid=361, compaction requested=true 2024-11-18T23:15:32,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:32,689 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:32,692 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:32,692 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:32,693 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,693 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d21039c119c440a987546cfb214849ad, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/8c4cf04d97da484d8eafd3b0196f6e44, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fde22ec320a740b88b4ba2740b8d5d1a] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.6 K 2024-11-18T23:15:32,694 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d21039c119c440a987546cfb214849ad, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731971728620 2024-11-18T23:15:32,694 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c4cf04d97da484d8eafd3b0196f6e44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731971729977 2024-11-18T23:15:32,694 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting fde22ec320a740b88b4ba2740b8d5d1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731971731405 2024-11-18T23:15:32,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-18T23:15:32,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,702 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T23:15:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:32,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:32,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:32,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:32,712 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:32,713 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:32,713 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:32,714 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:32,714 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:32,714 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/05b7d9bbad1f4ef09193a911e1f87138, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/bc2f276dfc1741ab91c559905fa4fe05, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f0f084cb1c4923831fa540ca66c1a8] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.6 K 2024-11-18T23:15:32,714 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/fa07bb608f3f4926bf24f7d570311970 is 50, key is test_row_0/A:col10/1731971731416/Put/seqid=0 2024-11-18T23:15:32,714 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05b7d9bbad1f4ef09193a911e1f87138, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731971728620 2024-11-18T23:15:32,716 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc2f276dfc1741ab91c559905fa4fe05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731971729977 2024-11-18T23:15:32,716 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3f0f084cb1c4923831fa540ca66c1a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731971731405 2024-11-18T23:15:32,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/f860683a3f0f4db2bddfb27a569450a7 is 50, key is test_row_0/A:col10/1731971731701/Put/seqid=0 2024-11-18T23:15:32,786 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:32,787 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/a31447dcf90d41e990663bc2642e4036 is 50, key is test_row_0/B:col10/1731971731416/Put/seqid=0 2024-11-18T23:15:32,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741894_1070 (size=12949) 2024-11-18T23:15:32,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741895_1071 (size=12301) 2024-11-18T23:15:32,808 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/f860683a3f0f4db2bddfb27a569450a7 2024-11-18T23:15:32,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741896_1072 (size=12949) 2024-11-18T23:15:32,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0c874638582c4455a70bf1a0bef2976a is 50, key is test_row_0/B:col10/1731971731701/Put/seqid=0 2024-11-18T23:15:32,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741897_1073 (size=12301) 2024-11-18T23:15:33,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:33,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:33,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971793061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971793071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971793078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971793088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T23:15:33,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971793200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971793200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971793208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971793218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,228 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/fa07bb608f3f4926bf24f7d570311970 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fa07bb608f3f4926bf24f7d570311970 2024-11-18T23:15:33,244 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/a31447dcf90d41e990663bc2642e4036 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/a31447dcf90d41e990663bc2642e4036 2024-11-18T23:15:33,252 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into fa07bb608f3f4926bf24f7d570311970(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:33,252 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:33,252 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971732689; duration=0sec 2024-11-18T23:15:33,252 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:33,252 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:33,253 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:33,258 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into a31447dcf90d41e990663bc2642e4036(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:33,258 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:33,258 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971732708; duration=0sec 2024-11-18T23:15:33,259 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:33,259 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:33,262 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:33,262 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:33,262 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:33,262 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5617401f9b5f457c9d6cb7828b7904e5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/44d3c9169fba4edeb1bb10b2d92e619d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/79f7621c7bbb475996ff48c44b3d8b9d] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.6 K 2024-11-18T23:15:33,268 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 5617401f9b5f457c9d6cb7828b7904e5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731971728620 2024-11-18T23:15:33,269 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 44d3c9169fba4edeb1bb10b2d92e619d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731971729977 2024-11-18T23:15:33,269 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 79f7621c7bbb475996ff48c44b3d8b9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731971731405 2024-11-18T23:15:33,297 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#61 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:33,298 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/9b3b8c9baacb4f7986e4d4a5dcf107d3 is 50, key is test_row_0/C:col10/1731971731416/Put/seqid=0 2024-11-18T23:15:33,305 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0c874638582c4455a70bf1a0bef2976a 2024-11-18T23:15:33,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741898_1074 (size=12949) 2024-11-18T23:15:33,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/867a2da11ab44bc8a7f173bb5e21e9e2 is 50, key is test_row_0/C:col10/1731971731701/Put/seqid=0 2024-11-18T23:15:33,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741899_1075 (size=12301) 2024-11-18T23:15:33,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971793420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971793420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,436 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971793428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971793440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971793741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,745 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971793747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971793747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:33,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971793768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:33,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:33,797 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/9b3b8c9baacb4f7986e4d4a5dcf107d3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9b3b8c9baacb4f7986e4d4a5dcf107d3 2024-11-18T23:15:33,826 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/867a2da11ab44bc8a7f173bb5e21e9e2 2024-11-18T23:15:33,846 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into 9b3b8c9baacb4f7986e4d4a5dcf107d3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:33,846 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:33,846 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971732709; duration=0sec 2024-11-18T23:15:33,847 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:33,847 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:33,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/f860683a3f0f4db2bddfb27a569450a7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/f860683a3f0f4db2bddfb27a569450a7 2024-11-18T23:15:33,872 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/f860683a3f0f4db2bddfb27a569450a7, entries=150, sequenceid=397, filesize=12.0 K 2024-11-18T23:15:33,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0c874638582c4455a70bf1a0bef2976a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0c874638582c4455a70bf1a0bef2976a 2024-11-18T23:15:33,880 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0c874638582c4455a70bf1a0bef2976a, entries=150, sequenceid=397, filesize=12.0 K 2024-11-18T23:15:33,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/867a2da11ab44bc8a7f173bb5e21e9e2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/867a2da11ab44bc8a7f173bb5e21e9e2 2024-11-18T23:15:33,891 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/867a2da11ab44bc8a7f173bb5e21e9e2, entries=150, sequenceid=397, filesize=12.0 K 2024-11-18T23:15:33,896 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1193ms, sequenceid=397, compaction requested=false 2024-11-18T23:15:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:33,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:33,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-18T23:15:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-18T23:15:33,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-18T23:15:33,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8590 sec 2024-11-18T23:15:33,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8940 sec 2024-11-18T23:15:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-18T23:15:34,189 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:34,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-18T23:15:34,193 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T23:15:34,194 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:34,196 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:34,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T23:15:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:34,264 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:34,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:34,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/5269bd890bef454a88d32041043b2785 is 50, key is test_row_0/A:col10/1731971733078/Put/seqid=0 2024-11-18T23:15:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T23:15:34,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741900_1076 (size=12301) 2024-11-18T23:15:34,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971794326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971794335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971794336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971794338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971794338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:34,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:34,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971794447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,449 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971794447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971794458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971794458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971794458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T23:15:34,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971794658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971794660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971794669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971794668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:34,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971794679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/5269bd890bef454a88d32041043b2785 2024-11-18T23:15:34,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0adae7f4a2d649fc86a9070359c21e58 is 50, key is test_row_0/B:col10/1731971733078/Put/seqid=0 2024-11-18T23:15:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T23:15:34,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:34,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741901_1077 (size=12301) 2024-11-18T23:15:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971794968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971794971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:34,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:34,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971794977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:34,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971794980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971795001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:35,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0adae7f4a2d649fc86a9070359c21e58 2024-11-18T23:15:35,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/41bc96205336460588d94754d1056574 is 50, key is test_row_0/C:col10/1731971733078/Put/seqid=0 2024-11-18T23:15:35,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741902_1078 (size=12301) 2024-11-18T23:15:35,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/41bc96205336460588d94754d1056574 2024-11-18T23:15:35,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:35,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:35,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:35,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:35,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:35,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/5269bd890bef454a88d32041043b2785 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/5269bd890bef454a88d32041043b2785 2024-11-18T23:15:35,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T23:15:35,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/5269bd890bef454a88d32041043b2785, entries=150, sequenceid=412, filesize=12.0 K 2024-11-18T23:15:35,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/0adae7f4a2d649fc86a9070359c21e58 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0adae7f4a2d649fc86a9070359c21e58 2024-11-18T23:15:35,378 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0adae7f4a2d649fc86a9070359c21e58, entries=150, sequenceid=412, filesize=12.0 K 2024-11-18T23:15:35,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/41bc96205336460588d94754d1056574 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/41bc96205336460588d94754d1056574 2024-11-18T23:15:35,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/41bc96205336460588d94754d1056574, entries=150, sequenceid=412, filesize=12.0 K 2024-11-18T23:15:35,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1153ms, sequenceid=412, compaction requested=true 2024-11-18T23:15:35,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:35,417 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:35,418 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:35,419 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:35,419 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,419 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fa07bb608f3f4926bf24f7d570311970, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/f860683a3f0f4db2bddfb27a569450a7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/5269bd890bef454a88d32041043b2785] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.7 K 2024-11-18T23:15:35,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:35,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:35,419 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:35,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:35,424 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa07bb608f3f4926bf24f7d570311970, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731971731405 2024-11-18T23:15:35,428 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:35,428 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:35,429 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,429 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/a31447dcf90d41e990663bc2642e4036, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0c874638582c4455a70bf1a0bef2976a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0adae7f4a2d649fc86a9070359c21e58] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.7 K 2024-11-18T23:15:35,429 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f860683a3f0f4db2bddfb27a569450a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731971731699 2024-11-18T23:15:35,430 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a31447dcf90d41e990663bc2642e4036, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731971731405 2024-11-18T23:15:35,431 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c874638582c4455a70bf1a0bef2976a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731971731699 2024-11-18T23:15:35,431 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5269bd890bef454a88d32041043b2785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731971733052 2024-11-18T23:15:35,440 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 0adae7f4a2d649fc86a9070359c21e58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731971733052 2024-11-18T23:15:35,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-18T23:15:35,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,461 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T23:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:35,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/349e2d0588e74954b1720399074e290b is 50, key is test_row_0/A:col10/1731971734328/Put/seqid=0 2024-11-18T23:15:35,510 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:35,511 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/367ba7aaee694d73a724935062407a60 is 50, key is test_row_0/A:col10/1731971733078/Put/seqid=0 2024-11-18T23:15:35,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:35,530 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#68 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:35,531 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/2d913c3b99804e7cadebfbe498d4898b is 50, key is test_row_0/B:col10/1731971733078/Put/seqid=0 2024-11-18T23:15:35,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741903_1079 (size=12301) 2024-11-18T23:15:35,560 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/349e2d0588e74954b1720399074e290b 2024-11-18T23:15:35,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/b431aa3344c34a38801d4e60a39368cd is 50, key is test_row_0/B:col10/1731971734328/Put/seqid=0 2024-11-18T23:15:35,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741904_1080 (size=13051) 2024-11-18T23:15:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971795574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971795581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741905_1081 (size=13051) 2024-11-18T23:15:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971795589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,609 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,629 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/367ba7aaee694d73a724935062407a60 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/367ba7aaee694d73a724935062407a60 2024-11-18T23:15:35,636 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/2d913c3b99804e7cadebfbe498d4898b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/2d913c3b99804e7cadebfbe498d4898b 2024-11-18T23:15:35,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971795599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741906_1082 (size=12301) 2024-11-18T23:15:35,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971795602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,661 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/b431aa3344c34a38801d4e60a39368cd 2024-11-18T23:15:35,662 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into 2d913c3b99804e7cadebfbe498d4898b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:35,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,662 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:35,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,662 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971735419; duration=0sec 2024-11-18T23:15:35,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,662 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:35,662 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:35,662 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:35,669 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into 367ba7aaee694d73a724935062407a60(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:35,669 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:35,669 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:35,669 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971735417; duration=0sec 2024-11-18T23:15:35,669 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:35,669 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,670 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:35,670 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:35,670 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9b3b8c9baacb4f7986e4d4a5dcf107d3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/867a2da11ab44bc8a7f173bb5e21e9e2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/41bc96205336460588d94754d1056574] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.7 K 2024-11-18T23:15:35,675 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b3b8c9baacb4f7986e4d4a5dcf107d3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731971731405 2024-11-18T23:15:35,677 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 867a2da11ab44bc8a7f173bb5e21e9e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1731971731699 2024-11-18T23:15:35,685 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 41bc96205336460588d94754d1056574, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731971733052 2024-11-18T23:15:35,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/6b1def59da5544948627138bf1f90826 is 50, key is test_row_0/C:col10/1731971734328/Put/seqid=0 2024-11-18T23:15:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971795707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971795721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971795721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,746 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:35,747 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/3208338eb7034138be947f6dcf68b670 is 50, key is test_row_0/C:col10/1731971733078/Put/seqid=0 2024-11-18T23:15:35,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971795760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:35,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971795773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:35,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:35,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741907_1083 (size=12301) 2024-11-18T23:15:35,796 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/6b1def59da5544948627138bf1f90826 2024-11-18T23:15:35,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741908_1084 (size=13051) 2024-11-18T23:15:35,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/349e2d0588e74954b1720399074e290b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/349e2d0588e74954b1720399074e290b 2024-11-18T23:15:35,845 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/3208338eb7034138be947f6dcf68b670 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3208338eb7034138be947f6dcf68b670 2024-11-18T23:15:35,850 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/349e2d0588e74954b1720399074e290b, entries=150, sequenceid=432, filesize=12.0 K 2024-11-18T23:15:35,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/b431aa3344c34a38801d4e60a39368cd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/b431aa3344c34a38801d4e60a39368cd 2024-11-18T23:15:35,866 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into 3208338eb7034138be947f6dcf68b670(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:35,866 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:35,866 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971735420; duration=0sec 2024-11-18T23:15:35,867 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:35,867 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:35,873 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/b431aa3344c34a38801d4e60a39368cd, entries=150, sequenceid=432, filesize=12.0 K 2024-11-18T23:15:35,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/6b1def59da5544948627138bf1f90826 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/6b1def59da5544948627138bf1f90826 2024-11-18T23:15:35,896 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/6b1def59da5544948627138bf1f90826, entries=150, sequenceid=432, filesize=12.0 K 2024-11-18T23:15:35,901 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 14ab88ca9857fdcfce5643ad2ad0a965 in 440ms, sequenceid=432, compaction requested=false 2024-11-18T23:15:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:35,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-18T23:15:35,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-18T23:15:35,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-18T23:15:35,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7070 sec 2024-11-18T23:15:35,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7180 sec 2024-11-18T23:15:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:35,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T23:15:35,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:35,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:35,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:35,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:35,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:35,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:35,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/1f4a5501e68042d58548a09813cf75ff is 50, key is test_row_0/A:col10/1731971735595/Put/seqid=0 2024-11-18T23:15:36,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741909_1085 (size=12301) 2024-11-18T23:15:36,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971796009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971796029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971796029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971796027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,036 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971796035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971796140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971796141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971796149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971796152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971796161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-18T23:15:36,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:36,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-18T23:15:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T23:15:36,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971796357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,368 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:36,376 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:36,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:36,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971796360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971796376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971796376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971796384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,386 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/1f4a5501e68042d58548a09813cf75ff 2024-11-18T23:15:36,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/72ba5c9382f94e18a5c9488e13d69dba is 50, key is test_row_0/B:col10/1731971735595/Put/seqid=0 2024-11-18T23:15:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T23:15:36,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741910_1086 (size=12301) 2024-11-18T23:15:36,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T23:15:36,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:36,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:36,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:36,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T23:15:36,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971796677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971796688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971796700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T23:15:36,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:36,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:36,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:36,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971796706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971796706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T23:15:36,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:36,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:36,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/72ba5c9382f94e18a5c9488e13d69dba 2024-11-18T23:15:36,972 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/67e92f8a05694bbfb5942d01a624e856 is 50, key is test_row_0/C:col10/1731971735595/Put/seqid=0 2024-11-18T23:15:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T23:15:37,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741911_1087 (size=12301) 2024-11-18T23:15:37,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/67e92f8a05694bbfb5942d01a624e856 2024-11-18T23:15:37,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T23:15:37,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:37,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:37,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:37,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:37,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/1f4a5501e68042d58548a09813cf75ff as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/1f4a5501e68042d58548a09813cf75ff 2024-11-18T23:15:37,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/1f4a5501e68042d58548a09813cf75ff, entries=150, sequenceid=450, filesize=12.0 K 2024-11-18T23:15:37,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/72ba5c9382f94e18a5c9488e13d69dba as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/72ba5c9382f94e18a5c9488e13d69dba 2024-11-18T23:15:37,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:37,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971797200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:37,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:37,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971797208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971797209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:37,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/72ba5c9382f94e18a5c9488e13d69dba, entries=150, sequenceid=450, filesize=12.0 K 2024-11-18T23:15:37,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T23:15:37,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/67e92f8a05694bbfb5942d01a624e856 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/67e92f8a05694bbfb5942d01a624e856 2024-11-18T23:15:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:37,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:37,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:37,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971797218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,243 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:37,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:37,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971797220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:37,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:37,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/67e92f8a05694bbfb5942d01a624e856, entries=150, sequenceid=450, filesize=12.0 K 2024-11-18T23:15:37,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1366ms, sequenceid=450, compaction requested=true 2024-11-18T23:15:37,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:37,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:37,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:37,315 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:37,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:37,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:37,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:37,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:37,320 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:37,321 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:37,321 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:37,321 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,321 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/367ba7aaee694d73a724935062407a60, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/349e2d0588e74954b1720399074e290b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/1f4a5501e68042d58548a09813cf75ff] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.8 K 2024-11-18T23:15:37,329 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 367ba7aaee694d73a724935062407a60, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731971733052 2024-11-18T23:15:37,333 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 349e2d0588e74954b1720399074e290b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731971734328 2024-11-18T23:15:37,336 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:37,336 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:37,337 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,337 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4a5501e68042d58548a09813cf75ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731971735595 2024-11-18T23:15:37,337 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/2d913c3b99804e7cadebfbe498d4898b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/b431aa3344c34a38801d4e60a39368cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/72ba5c9382f94e18a5c9488e13d69dba] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.8 K 2024-11-18T23:15:37,348 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d913c3b99804e7cadebfbe498d4898b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731971733052 2024-11-18T23:15:37,356 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b431aa3344c34a38801d4e60a39368cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731971734328 2024-11-18T23:15:37,364 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72ba5c9382f94e18a5c9488e13d69dba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731971735595 2024-11-18T23:15:37,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:37,385 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/c5df5754dcb740d6b5c564dabcdf3df4 is 50, key is test_row_0/A:col10/1731971735595/Put/seqid=0 2024-11-18T23:15:37,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-18T23:15:37,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,392 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T23:15:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:37,422 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:37,423 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/fe7316b2a546466a8cfae5a225bd156a is 50, key is test_row_0/B:col10/1731971735595/Put/seqid=0 2024-11-18T23:15:37,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/cbd4f51771d14d988c51f7c3cfd9a7d3 is 50, key is test_row_0/A:col10/1731971736022/Put/seqid=0 2024-11-18T23:15:37,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741912_1088 (size=13153) 2024-11-18T23:15:37,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741913_1089 (size=13153) 2024-11-18T23:15:37,479 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/c5df5754dcb740d6b5c564dabcdf3df4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/c5df5754dcb740d6b5c564dabcdf3df4 2024-11-18T23:15:37,496 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into c5df5754dcb740d6b5c564dabcdf3df4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:37,496 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:37,496 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971737315; duration=0sec 2024-11-18T23:15:37,497 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:37,497 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:37,497 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T23:15:37,498 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:37,498 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:37,498 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:37,499 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3208338eb7034138be947f6dcf68b670, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/6b1def59da5544948627138bf1f90826, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/67e92f8a05694bbfb5942d01a624e856] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.8 K 2024-11-18T23:15:37,501 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3208338eb7034138be947f6dcf68b670, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1731971733052 2024-11-18T23:15:37,501 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b1def59da5544948627138bf1f90826, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=432, earliestPutTs=1731971734328 2024-11-18T23:15:37,502 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e92f8a05694bbfb5942d01a624e856, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731971735595 2024-11-18T23:15:37,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741914_1090 (size=12301) 2024-11-18T23:15:37,507 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/fe7316b2a546466a8cfae5a225bd156a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/fe7316b2a546466a8cfae5a225bd156a 2024-11-18T23:15:37,510 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/cbd4f51771d14d988c51f7c3cfd9a7d3 2024-11-18T23:15:37,523 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:37,524 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/8da2312ba6f94cf89dad87df76e95aa0 is 50, key is test_row_0/C:col10/1731971735595/Put/seqid=0 2024-11-18T23:15:37,529 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into fe7316b2a546466a8cfae5a225bd156a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:37,529 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:37,529 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971737315; duration=0sec 2024-11-18T23:15:37,537 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:37,537 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:37,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/02df34b75ad64c7b918bdb200c12da21 is 50, key is test_row_0/B:col10/1731971736022/Put/seqid=0 2024-11-18T23:15:37,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741916_1092 (size=12301) 2024-11-18T23:15:37,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741915_1091 (size=13153) 2024-11-18T23:15:38,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/02df34b75ad64c7b918bdb200c12da21 2024-11-18T23:15:38,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/9a2614c11569478f88add7ddb789032a is 50, key is test_row_0/C:col10/1731971736022/Put/seqid=0 2024-11-18T23:15:38,051 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/8da2312ba6f94cf89dad87df76e95aa0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/8da2312ba6f94cf89dad87df76e95aa0 2024-11-18T23:15:38,064 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into 8da2312ba6f94cf89dad87df76e95aa0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:38,064 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:38,064 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971737315; duration=0sec 2024-11-18T23:15:38,064 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:38,064 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:38,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741917_1093 (size=12301) 2024-11-18T23:15:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:38,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. as already flushing 2024-11-18T23:15:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36746 deadline: 1731971798260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,274 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 14ab88ca9857fdcfce5643ad2ad0a965 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:15:38,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,275 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971798262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971798274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971798285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971798261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971798387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971798388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971798408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971798408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,487 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/9a2614c11569478f88add7ddb789032a 2024-11-18T23:15:38,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/cbd4f51771d14d988c51f7c3cfd9a7d3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/cbd4f51771d14d988c51f7c3cfd9a7d3 2024-11-18T23:15:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T23:15:38,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/cbd4f51771d14d988c51f7c3cfd9a7d3, entries=150, sequenceid=468, filesize=12.0 K 2024-11-18T23:15:38,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/02df34b75ad64c7b918bdb200c12da21 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/02df34b75ad64c7b918bdb200c12da21 2024-11-18T23:15:38,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/02df34b75ad64c7b918bdb200c12da21, entries=150, sequenceid=468, filesize=12.0 K 2024-11-18T23:15:38,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/9a2614c11569478f88add7ddb789032a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9a2614c11569478f88add7ddb789032a 2024-11-18T23:15:38,542 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9a2614c11569478f88add7ddb789032a, entries=150, sequenceid=468, filesize=12.0 K 2024-11-18T23:15:38,547 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1154ms, sequenceid=468, compaction requested=false 2024-11-18T23:15:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-18T23:15:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-18T23:15:38,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-18T23:15:38,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1780 sec 2024-11-18T23:15:38,603 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.2470 sec 2024-11-18T23:15:38,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:15:38,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:38,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:38,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:38,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:38,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:38,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/bd83fc3d82164654b8af34e81520832e is 50, key is test_row_0/A:col10/1731971738600/Put/seqid=0 2024-11-18T23:15:38,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741918_1094 (size=14741) 2024-11-18T23:15:38,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/bd83fc3d82164654b8af34e81520832e 2024-11-18T23:15:38,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/be51c285b7544c5babe0a5e081e135ce is 50, key is test_row_0/B:col10/1731971738600/Put/seqid=0 2024-11-18T23:15:38,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971798734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971798747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971798753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971798754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741919_1095 (size=12301) 2024-11-18T23:15:38,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/be51c285b7544c5babe0a5e081e135ce 2024-11-18T23:15:38,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/0c43cbeea1a64616b9207e201e02851f is 50, key is test_row_0/C:col10/1731971738600/Put/seqid=0 2024-11-18T23:15:38,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741920_1096 (size=12301) 2024-11-18T23:15:38,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971798857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971798869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/0c43cbeea1a64616b9207e201e02851f 2024-11-18T23:15:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/bd83fc3d82164654b8af34e81520832e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/bd83fc3d82164654b8af34e81520832e 2024-11-18T23:15:38,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36774 deadline: 1731971798877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:38,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36796 deadline: 1731971798877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:38,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:38,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/bd83fc3d82164654b8af34e81520832e, entries=200, sequenceid=487, filesize=14.4 K 2024-11-18T23:15:38,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/be51c285b7544c5babe0a5e081e135ce as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/be51c285b7544c5babe0a5e081e135ce 2024-11-18T23:15:38,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/be51c285b7544c5babe0a5e081e135ce, entries=150, sequenceid=487, filesize=12.0 K 2024-11-18T23:15:38,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/0c43cbeea1a64616b9207e201e02851f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/0c43cbeea1a64616b9207e201e02851f 2024-11-18T23:15:38,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/0c43cbeea1a64616b9207e201e02851f, entries=150, sequenceid=487, filesize=12.0 K 2024-11-18T23:15:38,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 14ab88ca9857fdcfce5643ad2ad0a965 in 321ms, sequenceid=487, compaction requested=true 2024-11-18T23:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:38,930 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:38,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:38,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:38,933 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:38,949 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:38,949 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:38,949 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:38,949 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/c5df5754dcb740d6b5c564dabcdf3df4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/cbd4f51771d14d988c51f7c3cfd9a7d3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/bd83fc3d82164654b8af34e81520832e] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=39.3 K 2024-11-18T23:15:38,951 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:38,951 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5df5754dcb740d6b5c564dabcdf3df4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731971735595 2024-11-18T23:15:38,952 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:38,952 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:38,952 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/fe7316b2a546466a8cfae5a225bd156a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/02df34b75ad64c7b918bdb200c12da21, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/be51c285b7544c5babe0a5e081e135ce] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.9 K 2024-11-18T23:15:38,954 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbd4f51771d14d988c51f7c3cfd9a7d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731971736021 2024-11-18T23:15:38,955 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd83fc3d82164654b8af34e81520832e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731971738243 2024-11-18T23:15:38,955 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting fe7316b2a546466a8cfae5a225bd156a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731971735595 2024-11-18T23:15:38,964 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 02df34b75ad64c7b918bdb200c12da21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731971736021 2024-11-18T23:15:38,969 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting be51c285b7544c5babe0a5e081e135ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731971738243 2024-11-18T23:15:39,003 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:39,003 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/2df5fca7df19460f85c7a99439b1ce80 is 50, key is test_row_0/A:col10/1731971738600/Put/seqid=0 2024-11-18T23:15:39,023 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:39,024 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/cdcf569a75f044cbbc4fa46332b4320b is 50, key is test_row_0/B:col10/1731971738600/Put/seqid=0 2024-11-18T23:15:39,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741921_1097 (size=13255) 2024-11-18T23:15:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:39,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T23:15:39,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:39,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:39,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:39,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:39,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:39,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:39,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d9a5ad82a9bf4eb89fca09acbdae3daf is 50, key is test_row_0/A:col10/1731971739068/Put/seqid=0 2024-11-18T23:15:39,109 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/2df5fca7df19460f85c7a99439b1ce80 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2df5fca7df19460f85c7a99439b1ce80 2024-11-18T23:15:39,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741922_1098 (size=13255) 2024-11-18T23:15:39,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741923_1099 (size=12301) 2024-11-18T23:15:39,129 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into 2df5fca7df19460f85c7a99439b1ce80(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:39,129 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:39,130 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971738930; duration=0sec 2024-11-18T23:15:39,130 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:39,130 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:39,130 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:39,138 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:39,138 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/C is initiating minor compaction (all files) 2024-11-18T23:15:39,138 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/C in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:39,138 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/8da2312ba6f94cf89dad87df76e95aa0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9a2614c11569478f88add7ddb789032a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/0c43cbeea1a64616b9207e201e02851f] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=36.9 K 2024-11-18T23:15:39,139 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da2312ba6f94cf89dad87df76e95aa0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731971735595 2024-11-18T23:15:39,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d9a5ad82a9bf4eb89fca09acbdae3daf 2024-11-18T23:15:39,141 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a2614c11569478f88add7ddb789032a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731971736021 2024-11-18T23:15:39,142 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c43cbeea1a64616b9207e201e02851f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731971738243 2024-11-18T23:15:39,144 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/cdcf569a75f044cbbc4fa46332b4320b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/cdcf569a75f044cbbc4fa46332b4320b 2024-11-18T23:15:39,155 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into cdcf569a75f044cbbc4fa46332b4320b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:39,155 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:39,155 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971738930; duration=0sec 2024-11-18T23:15:39,156 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:39,156 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:39,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/4344321391734b49a638d97fc8d7dc1c is 50, key is test_row_0/B:col10/1731971739068/Put/seqid=0 2024-11-18T23:15:39,159 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-18T23:15:39,159 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-18T23:15:39,159 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,159 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,160 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-18T23:15:39,160 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:39,162 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-18T23:15:39,162 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971799137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:39,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:39,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971799139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:39,166 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,166 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,166 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,167 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,167 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,167 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,167 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,168 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:39,183 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-18T23:15:39,183 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,183 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,183 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,195 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-18T23:15:39,195 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,195 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,195 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741924_1100 (size=12301) 2024-11-18T23:15:39,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/4344321391734b49a638d97fc8d7dc1c 2024-11-18T23:15:39,210 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:39,211 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/25b8ea349de844278753035c9a9f4230 is 50, key is test_row_0/C:col10/1731971738600/Put/seqid=0 2024-11-18T23:15:39,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/211070b9abed48acb2cbd5e035ea2c4f is 50, key is test_row_0/C:col10/1731971739068/Put/seqid=0 2024-11-18T23:15:39,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741925_1101 (size=13255) 2024-11-18T23:15:39,250 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/25b8ea349de844278753035c9a9f4230 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/25b8ea349de844278753035c9a9f4230 2024-11-18T23:15:39,265 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/C of 14ab88ca9857fdcfce5643ad2ad0a965 into 25b8ea349de844278753035c9a9f4230(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:39,265 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:39,265 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/C, priority=13, startTime=1731971738931; duration=0sec 2024-11-18T23:15:39,265 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:39,265 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:39,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741926_1102 (size=12301) 2024-11-18T23:15:39,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971799272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:39,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971799281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:39,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:39,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36766 deadline: 1731971799483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:39,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:39,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36750 deadline: 1731971799488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:15:39,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/211070b9abed48acb2cbd5e035ea2c4f 2024-11-18T23:15:39,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/d9a5ad82a9bf4eb89fca09acbdae3daf as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d9a5ad82a9bf4eb89fca09acbdae3daf 2024-11-18T23:15:39,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d9a5ad82a9bf4eb89fca09acbdae3daf, entries=150, sequenceid=505, filesize=12.0 K 2024-11-18T23:15:39,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/4344321391734b49a638d97fc8d7dc1c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/4344321391734b49a638d97fc8d7dc1c 2024-11-18T23:15:39,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/4344321391734b49a638d97fc8d7dc1c, entries=150, sequenceid=505, filesize=12.0 K 2024-11-18T23:15:39,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/211070b9abed48acb2cbd5e035ea2c4f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/211070b9abed48acb2cbd5e035ea2c4f 2024-11-18T23:15:39,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/211070b9abed48acb2cbd5e035ea2c4f, entries=150, sequenceid=505, filesize=12.0 K 2024-11-18T23:15:39,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 14ab88ca9857fdcfce5643ad2ad0a965 in 638ms, sequenceid=505, compaction requested=false 2024-11-18T23:15:39,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:39,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:15:39,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:39,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:39,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:39,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:39,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:39,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:39,804 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-18T23:15:39,804 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,804 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,805 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,814 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-18T23:15:39,814 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:39,814 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,814 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:39,814 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:39,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/88ab5d50f046476da06f6b4e79eb0898 is 50, key is test_row_0/A:col10/1731971739800/Put/seqid=0 2024-11-18T23:15:39,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741927_1103 (size=12301) 2024-11-18T23:15:39,838 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/88ab5d50f046476da06f6b4e79eb0898 2024-11-18T23:15:39,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/553b49958dd24ffba82dec049ae210f2 is 50, key is test_row_0/B:col10/1731971739800/Put/seqid=0 2024-11-18T23:15:39,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741928_1104 (size=12301) 2024-11-18T23:15:40,259 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/553b49958dd24ffba82dec049ae210f2 2024-11-18T23:15:40,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/7b7096488d0f4d6ea0c4680618f6b66d is 50, key is test_row_0/C:col10/1731971739800/Put/seqid=0 2024-11-18T23:15:40,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741929_1105 (size=12301) 2024-11-18T23:15:40,288 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-18T23:15:40,289 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:15:40,289 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:40,289 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:40,289 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-18T23:15:40,538 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T23:15:40,538 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1225 2024-11-18T23:15:40,539 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1224 2024-11-18T23:15:40,539 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T23:15:40,539 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 606 2024-11-18T23:15:40,539 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1817 rows 2024-11-18T23:15:40,539 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 608 2024-11-18T23:15:40,539 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1823 rows 2024-11-18T23:15:40,539 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:15:40,539 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:15:40,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:40,539 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:40,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:15:40,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T23:15:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T23:15:40,553 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971740552"}]},"ts":"1731971740552"} 2024-11-18T23:15:40,555 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T23:15:40,555 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T23:15:40,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:15:40,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, UNASSIGN}] 2024-11-18T23:15:40,564 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, UNASSIGN 2024-11-18T23:15:40,566 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=14ab88ca9857fdcfce5643ad2ad0a965, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:40,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, UNASSIGN because future has completed 2024-11-18T23:15:40,569 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:15:40,570 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:15:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T23:15:40,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/7b7096488d0f4d6ea0c4680618f6b66d 2024-11-18T23:15:40,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/88ab5d50f046476da06f6b4e79eb0898 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/88ab5d50f046476da06f6b4e79eb0898 2024-11-18T23:15:40,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/88ab5d50f046476da06f6b4e79eb0898, entries=150, sequenceid=524, filesize=12.0 K 2024-11-18T23:15:40,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/553b49958dd24ffba82dec049ae210f2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/553b49958dd24ffba82dec049ae210f2 2024-11-18T23:15:40,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/553b49958dd24ffba82dec049ae210f2, entries=150, sequenceid=524, filesize=12.0 K 2024-11-18T23:15:40,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/7b7096488d0f4d6ea0c4680618f6b66d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/7b7096488d0f4d6ea0c4680618f6b66d 2024-11-18T23:15:40,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/7b7096488d0f4d6ea0c4680618f6b66d, entries=150, sequenceid=524, filesize=12.0 K 2024-11-18T23:15:40,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for 14ab88ca9857fdcfce5643ad2ad0a965 in 914ms, sequenceid=524, compaction requested=true 2024-11-18T23:15:40,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:40,717 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 14ab88ca9857fdcfce5643ad2ad0a965:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:40,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:40,717 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:40,719 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:40,719 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/A is initiating minor compaction (all files) 2024-11-18T23:15:40,719 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/A in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:40,719 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2df5fca7df19460f85c7a99439b1ce80, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d9a5ad82a9bf4eb89fca09acbdae3daf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/88ab5d50f046476da06f6b4e79eb0898] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=37.0 K 2024-11-18T23:15:40,719 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:40,719 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 14ab88ca9857fdcfce5643ad2ad0a965/B is initiating minor compaction (all files) 2024-11-18T23:15:40,720 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 14ab88ca9857fdcfce5643ad2ad0a965/B in TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:40,720 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/cdcf569a75f044cbbc4fa46332b4320b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/4344321391734b49a638d97fc8d7dc1c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/553b49958dd24ffba82dec049ae210f2] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp, totalSize=37.0 K 2024-11-18T23:15:40,720 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2df5fca7df19460f85c7a99439b1ce80, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731971738243 2024-11-18T23:15:40,721 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting cdcf569a75f044cbbc4fa46332b4320b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731971738243 2024-11-18T23:15:40,722 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9a5ad82a9bf4eb89fca09acbdae3daf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731971738736 2024-11-18T23:15:40,722 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4344321391734b49a638d97fc8d7dc1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=505, earliestPutTs=1731971738736 2024-11-18T23:15:40,722 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88ab5d50f046476da06f6b4e79eb0898, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1731971739096 2024-11-18T23:15:40,723 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 553b49958dd24ffba82dec049ae210f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1731971739096 2024-11-18T23:15:40,737 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(122): Close 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:40,737 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:15:40,738 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1722): Closing 14ab88ca9857fdcfce5643ad2ad0a965, disabling compactions & flushes 2024-11-18T23:15:40,739 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:40,740 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#A#compaction#93 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:40,741 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/eef46b094a9c4035bbd0c6db2d35b9c9 is 50, key is test_row_0/A:col10/1731971739800/Put/seqid=0 2024-11-18T23:15:40,787 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 14ab88ca9857fdcfce5643ad2ad0a965#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:40,788 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/5e7e8d5a495f4d6c952b3a08205eed23 is 50, key is test_row_0/B:col10/1731971739800/Put/seqid=0 2024-11-18T23:15:40,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741930_1106 (size=13357) 2024-11-18T23:15:40,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741931_1107 (size=13357) 2024-11-18T23:15:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T23:15:40,903 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/5e7e8d5a495f4d6c952b3a08205eed23 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5e7e8d5a495f4d6c952b3a08205eed23 2024-11-18T23:15:40,961 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/B of 14ab88ca9857fdcfce5643ad2ad0a965 into 5e7e8d5a495f4d6c952b3a08205eed23(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:40,961 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:40,961 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/B, priority=13, startTime=1731971740717; duration=0sec 2024-11-18T23:15:40,961 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:40,962 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:B 2024-11-18T23:15:40,962 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. because compaction request was cancelled 2024-11-18T23:15:40,962 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:C 2024-11-18T23:15:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T23:15:41,198 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/eef46b094a9c4035bbd0c6db2d35b9c9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/eef46b094a9c4035bbd0c6db2d35b9c9 2024-11-18T23:15:41,205 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 14ab88ca9857fdcfce5643ad2ad0a965/A of 14ab88ca9857fdcfce5643ad2ad0a965 into eef46b094a9c4035bbd0c6db2d35b9c9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:41,205 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 14ab88ca9857fdcfce5643ad2ad0a965: 2024-11-18T23:15:41,205 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., storeName=14ab88ca9857fdcfce5643ad2ad0a965/A, priority=13, startTime=1731971740717; duration=0sec 2024-11-18T23:15:41,205 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:41,205 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:41,205 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:41,205 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 14ab88ca9857fdcfce5643ad2ad0a965:A 2024-11-18T23:15:41,205 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. after waiting 0 ms 2024-11-18T23:15:41,205 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:41,206 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(2902): Flushing 14ab88ca9857fdcfce5643ad2ad0a965 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-18T23:15:41,206 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=A 2024-11-18T23:15:41,206 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:41,206 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=B 2024-11-18T23:15:41,206 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:41,206 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 14ab88ca9857fdcfce5643ad2ad0a965, store=C 2024-11-18T23:15:41,206 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:41,211 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/3badfcab0e484b1da02fd6104b3fcd5a is 50, key is test_row_1/A:col10/1731971740287/Put/seqid=0 2024-11-18T23:15:41,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741932_1108 (size=9857) 2024-11-18T23:15:41,465 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T23:15:41,621 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/3badfcab0e484b1da02fd6104b3fcd5a 2024-11-18T23:15:41,635 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/3091d6c14315491fb073892af9392296 is 50, key is test_row_1/B:col10/1731971740287/Put/seqid=0 2024-11-18T23:15:41,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741933_1109 (size=9857) 2024-11-18T23:15:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T23:15:42,055 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/3091d6c14315491fb073892af9392296 2024-11-18T23:15:42,064 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/2283067472ce495488f7bbaa4aaa5336 is 50, key is test_row_1/C:col10/1731971740287/Put/seqid=0 2024-11-18T23:15:42,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741934_1110 (size=9857) 2024-11-18T23:15:42,470 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/2283067472ce495488f7bbaa4aaa5336 2024-11-18T23:15:42,477 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/A/3badfcab0e484b1da02fd6104b3fcd5a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/3badfcab0e484b1da02fd6104b3fcd5a 2024-11-18T23:15:42,483 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/3badfcab0e484b1da02fd6104b3fcd5a, entries=100, sequenceid=531, filesize=9.6 K 2024-11-18T23:15:42,484 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/B/3091d6c14315491fb073892af9392296 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3091d6c14315491fb073892af9392296 2024-11-18T23:15:42,491 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3091d6c14315491fb073892af9392296, entries=100, sequenceid=531, filesize=9.6 K 2024-11-18T23:15:42,492 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/.tmp/C/2283067472ce495488f7bbaa4aaa5336 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/2283067472ce495488f7bbaa4aaa5336 2024-11-18T23:15:42,498 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/2283067472ce495488f7bbaa4aaa5336, entries=100, sequenceid=531, filesize=9.6 K 2024-11-18T23:15:42,499 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1294ms, sequenceid=531, compaction requested=true 2024-11-18T23:15:42,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e5df1229d534a119e242f67e54bed34, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d0ae6dfc3f03411cb11317b93e9ca4ea, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e4bb35047d8474eaf4d94c1ae228569, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/019c3687352f427894677fe7c52bf1d6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/0b2369b1b21e474c8fe17be7f4994220, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/7d57663ac1d9459ab37b0f9ef9d89dd2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2ad5ab0addd94697ab85982ba3a41afd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/96f04931c031483a86c8429cf344271c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/40eeca82e5a04924a5c2e9fa9da85f49, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d27f752edf33427aa31c794fda3ffc01, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/6d9dfa8fe2744ea5abd835e3919f6982, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/431ae34b26264e34a7e1af9df46b2130, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/07925b3fd2c04428aae0cc95a90514a6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/80c9e5fc87244028aa2af67da4484d32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/86a5a2745d0543e6adff0e207f1cb596, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/81cfea96921e46c79ad08795a6397255, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d21039c119c440a987546cfb214849ad, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/8c4cf04d97da484d8eafd3b0196f6e44, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fa07bb608f3f4926bf24f7d570311970, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fde22ec320a740b88b4ba2740b8d5d1a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/f860683a3f0f4db2bddfb27a569450a7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/367ba7aaee694d73a724935062407a60, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/5269bd890bef454a88d32041043b2785, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/349e2d0588e74954b1720399074e290b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/c5df5754dcb740d6b5c564dabcdf3df4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/1f4a5501e68042d58548a09813cf75ff, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/cbd4f51771d14d988c51f7c3cfd9a7d3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/bd83fc3d82164654b8af34e81520832e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2df5fca7df19460f85c7a99439b1ce80, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d9a5ad82a9bf4eb89fca09acbdae3daf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/88ab5d50f046476da06f6b4e79eb0898] to archive 2024-11-18T23:15:42,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:15:42,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e5df1229d534a119e242f67e54bed34 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e5df1229d534a119e242f67e54bed34 2024-11-18T23:15:42,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d0ae6dfc3f03411cb11317b93e9ca4ea to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d0ae6dfc3f03411cb11317b93e9ca4ea 2024-11-18T23:15:42,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e4bb35047d8474eaf4d94c1ae228569 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/9e4bb35047d8474eaf4d94c1ae228569 2024-11-18T23:15:42,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/019c3687352f427894677fe7c52bf1d6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/019c3687352f427894677fe7c52bf1d6 2024-11-18T23:15:42,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/0b2369b1b21e474c8fe17be7f4994220 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/0b2369b1b21e474c8fe17be7f4994220 2024-11-18T23:15:42,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/7d57663ac1d9459ab37b0f9ef9d89dd2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/7d57663ac1d9459ab37b0f9ef9d89dd2 2024-11-18T23:15:42,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2ad5ab0addd94697ab85982ba3a41afd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2ad5ab0addd94697ab85982ba3a41afd 2024-11-18T23:15:42,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/96f04931c031483a86c8429cf344271c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/96f04931c031483a86c8429cf344271c 2024-11-18T23:15:42,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/40eeca82e5a04924a5c2e9fa9da85f49 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/40eeca82e5a04924a5c2e9fa9da85f49 2024-11-18T23:15:42,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d27f752edf33427aa31c794fda3ffc01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d27f752edf33427aa31c794fda3ffc01 2024-11-18T23:15:42,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/6d9dfa8fe2744ea5abd835e3919f6982 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/6d9dfa8fe2744ea5abd835e3919f6982 2024-11-18T23:15:42,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/431ae34b26264e34a7e1af9df46b2130 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/431ae34b26264e34a7e1af9df46b2130 2024-11-18T23:15:42,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/07925b3fd2c04428aae0cc95a90514a6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/07925b3fd2c04428aae0cc95a90514a6 2024-11-18T23:15:42,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/80c9e5fc87244028aa2af67da4484d32 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/80c9e5fc87244028aa2af67da4484d32 2024-11-18T23:15:42,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/86a5a2745d0543e6adff0e207f1cb596 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/86a5a2745d0543e6adff0e207f1cb596 2024-11-18T23:15:42,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/81cfea96921e46c79ad08795a6397255 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/81cfea96921e46c79ad08795a6397255 2024-11-18T23:15:42,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d21039c119c440a987546cfb214849ad to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d21039c119c440a987546cfb214849ad 2024-11-18T23:15:42,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/8c4cf04d97da484d8eafd3b0196f6e44 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/8c4cf04d97da484d8eafd3b0196f6e44 2024-11-18T23:15:42,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fa07bb608f3f4926bf24f7d570311970 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fa07bb608f3f4926bf24f7d570311970 2024-11-18T23:15:42,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fde22ec320a740b88b4ba2740b8d5d1a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/fde22ec320a740b88b4ba2740b8d5d1a 2024-11-18T23:15:42,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/f860683a3f0f4db2bddfb27a569450a7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/f860683a3f0f4db2bddfb27a569450a7 2024-11-18T23:15:42,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/367ba7aaee694d73a724935062407a60 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/367ba7aaee694d73a724935062407a60 2024-11-18T23:15:42,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/5269bd890bef454a88d32041043b2785 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/5269bd890bef454a88d32041043b2785 2024-11-18T23:15:42,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/349e2d0588e74954b1720399074e290b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/349e2d0588e74954b1720399074e290b 2024-11-18T23:15:42,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/c5df5754dcb740d6b5c564dabcdf3df4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/c5df5754dcb740d6b5c564dabcdf3df4 2024-11-18T23:15:42,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/1f4a5501e68042d58548a09813cf75ff to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/1f4a5501e68042d58548a09813cf75ff 2024-11-18T23:15:42,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/cbd4f51771d14d988c51f7c3cfd9a7d3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/cbd4f51771d14d988c51f7c3cfd9a7d3 2024-11-18T23:15:42,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/bd83fc3d82164654b8af34e81520832e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/bd83fc3d82164654b8af34e81520832e 2024-11-18T23:15:42,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2df5fca7df19460f85c7a99439b1ce80 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/2df5fca7df19460f85c7a99439b1ce80 2024-11-18T23:15:42,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d9a5ad82a9bf4eb89fca09acbdae3daf to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/d9a5ad82a9bf4eb89fca09acbdae3daf 2024-11-18T23:15:42,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/88ab5d50f046476da06f6b4e79eb0898 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/88ab5d50f046476da06f6b4e79eb0898 2024-11-18T23:15:42,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/7f9e10e037c9435a9f88f53043ae9688, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/9cb813bfebb54f65832b61820e5606c6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/c9ec51786f454dbea3ed17f36ebf3eaa, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3187ce08264e4028bb4d1e81f442acee, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ab9e012047e541e09172c8364978832d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e1b540d9aa9642f1b42d24149d5fb793, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f3ee3127e04832817be9d2e5d28e48, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5736c244bd2f40f39e3d7561f1463bf1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ccd258b186024b94a5b9f2db245241ad, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e75e12ee529a462296c39d439d1958a3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/6d2ae24f7b9141aebf655271314e199e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ce2fe14dbad4469486a6419d88fc677c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/eaa3172b04364be7957a14cb5a012b33, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e766fc433245472ab37df163fbfeccd1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0d7ebb331f2045ec8aea6418099f7965, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/05b7d9bbad1f4ef09193a911e1f87138, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/889a4279a14d4afe85710c48a303aad9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/bc2f276dfc1741ab91c559905fa4fe05, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/a31447dcf90d41e990663bc2642e4036, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f0f084cb1c4923831fa540ca66c1a8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0c874638582c4455a70bf1a0bef2976a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/2d913c3b99804e7cadebfbe498d4898b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0adae7f4a2d649fc86a9070359c21e58, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/b431aa3344c34a38801d4e60a39368cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/fe7316b2a546466a8cfae5a225bd156a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/72ba5c9382f94e18a5c9488e13d69dba, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/02df34b75ad64c7b918bdb200c12da21, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/cdcf569a75f044cbbc4fa46332b4320b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/be51c285b7544c5babe0a5e081e135ce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/4344321391734b49a638d97fc8d7dc1c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/553b49958dd24ffba82dec049ae210f2] to archive 2024-11-18T23:15:42,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:15:42,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/7f9e10e037c9435a9f88f53043ae9688 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/7f9e10e037c9435a9f88f53043ae9688 2024-11-18T23:15:42,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/9cb813bfebb54f65832b61820e5606c6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/9cb813bfebb54f65832b61820e5606c6 2024-11-18T23:15:42,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/c9ec51786f454dbea3ed17f36ebf3eaa to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/c9ec51786f454dbea3ed17f36ebf3eaa 2024-11-18T23:15:42,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3187ce08264e4028bb4d1e81f442acee to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3187ce08264e4028bb4d1e81f442acee 2024-11-18T23:15:42,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ab9e012047e541e09172c8364978832d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ab9e012047e541e09172c8364978832d 2024-11-18T23:15:42,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e1b540d9aa9642f1b42d24149d5fb793 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e1b540d9aa9642f1b42d24149d5fb793 2024-11-18T23:15:42,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f3ee3127e04832817be9d2e5d28e48 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f3ee3127e04832817be9d2e5d28e48 2024-11-18T23:15:42,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5736c244bd2f40f39e3d7561f1463bf1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5736c244bd2f40f39e3d7561f1463bf1 2024-11-18T23:15:42,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ccd258b186024b94a5b9f2db245241ad to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ccd258b186024b94a5b9f2db245241ad 2024-11-18T23:15:42,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e75e12ee529a462296c39d439d1958a3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e75e12ee529a462296c39d439d1958a3 2024-11-18T23:15:42,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/6d2ae24f7b9141aebf655271314e199e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/6d2ae24f7b9141aebf655271314e199e 2024-11-18T23:15:42,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ce2fe14dbad4469486a6419d88fc677c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/ce2fe14dbad4469486a6419d88fc677c 2024-11-18T23:15:42,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/eaa3172b04364be7957a14cb5a012b33 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/eaa3172b04364be7957a14cb5a012b33 2024-11-18T23:15:42,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e766fc433245472ab37df163fbfeccd1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/e766fc433245472ab37df163fbfeccd1 2024-11-18T23:15:42,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0d7ebb331f2045ec8aea6418099f7965 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0d7ebb331f2045ec8aea6418099f7965 2024-11-18T23:15:42,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/05b7d9bbad1f4ef09193a911e1f87138 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/05b7d9bbad1f4ef09193a911e1f87138 2024-11-18T23:15:42,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/889a4279a14d4afe85710c48a303aad9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/889a4279a14d4afe85710c48a303aad9 2024-11-18T23:15:42,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/bc2f276dfc1741ab91c559905fa4fe05 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/bc2f276dfc1741ab91c559905fa4fe05 2024-11-18T23:15:42,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/a31447dcf90d41e990663bc2642e4036 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/a31447dcf90d41e990663bc2642e4036 2024-11-18T23:15:42,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f0f084cb1c4923831fa540ca66c1a8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/f3f0f084cb1c4923831fa540ca66c1a8 2024-11-18T23:15:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T23:15:42,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0c874638582c4455a70bf1a0bef2976a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0c874638582c4455a70bf1a0bef2976a 2024-11-18T23:15:42,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/2d913c3b99804e7cadebfbe498d4898b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/2d913c3b99804e7cadebfbe498d4898b 2024-11-18T23:15:42,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0adae7f4a2d649fc86a9070359c21e58 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/0adae7f4a2d649fc86a9070359c21e58 2024-11-18T23:15:42,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/b431aa3344c34a38801d4e60a39368cd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/b431aa3344c34a38801d4e60a39368cd 2024-11-18T23:15:42,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/fe7316b2a546466a8cfae5a225bd156a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/fe7316b2a546466a8cfae5a225bd156a 2024-11-18T23:15:42,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/72ba5c9382f94e18a5c9488e13d69dba to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/72ba5c9382f94e18a5c9488e13d69dba 2024-11-18T23:15:42,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/02df34b75ad64c7b918bdb200c12da21 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/02df34b75ad64c7b918bdb200c12da21 2024-11-18T23:15:42,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/cdcf569a75f044cbbc4fa46332b4320b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/cdcf569a75f044cbbc4fa46332b4320b 2024-11-18T23:15:42,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/be51c285b7544c5babe0a5e081e135ce to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/be51c285b7544c5babe0a5e081e135ce 2024-11-18T23:15:42,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/4344321391734b49a638d97fc8d7dc1c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/4344321391734b49a638d97fc8d7dc1c 2024-11-18T23:15:42,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/553b49958dd24ffba82dec049ae210f2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/553b49958dd24ffba82dec049ae210f2 2024-11-18T23:15:42,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/474a0e5bca464b9b963500855a3cc2e7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/737c1d2915324a30977c79549e031d1b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/dad670b36c9e4461b390711a9e2043ac, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/342b74f301c041a59fcbbf63a72151e8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d19ef0ba41204317bddee5f7b95d3c1a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/e5e3bf6598154603becb0e42be0de2a0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/06ee867e65f04749943c1f2f21542451, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3c2c6540d04f4068bcd68c91b284feb2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/349b88ab90de44b08883a45e003e8a32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d8da56a8f3ab4437a4948d4a00171c32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4357ef97862843b0a7f8d93a88ce67c0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/fd46b45aa4974d289b6618cee25c87c4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/f455ff19147e4627b7a210597680354d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4e9f6621b02a4124a77e71751567888d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5617401f9b5f457c9d6cb7828b7904e5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/35f7bd1096fb416190ad932a204f6157, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/44d3c9169fba4edeb1bb10b2d92e619d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9b3b8c9baacb4f7986e4d4a5dcf107d3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/79f7621c7bbb475996ff48c44b3d8b9d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/867a2da11ab44bc8a7f173bb5e21e9e2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3208338eb7034138be947f6dcf68b670, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/41bc96205336460588d94754d1056574, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/6b1def59da5544948627138bf1f90826, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/8da2312ba6f94cf89dad87df76e95aa0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/67e92f8a05694bbfb5942d01a624e856, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9a2614c11569478f88add7ddb789032a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/0c43cbeea1a64616b9207e201e02851f] to archive 2024-11-18T23:15:42,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:15:42,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/474a0e5bca464b9b963500855a3cc2e7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/474a0e5bca464b9b963500855a3cc2e7 2024-11-18T23:15:42,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/737c1d2915324a30977c79549e031d1b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/737c1d2915324a30977c79549e031d1b 2024-11-18T23:15:42,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/dad670b36c9e4461b390711a9e2043ac to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/dad670b36c9e4461b390711a9e2043ac 2024-11-18T23:15:42,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/342b74f301c041a59fcbbf63a72151e8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/342b74f301c041a59fcbbf63a72151e8 2024-11-18T23:15:42,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d19ef0ba41204317bddee5f7b95d3c1a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d19ef0ba41204317bddee5f7b95d3c1a 2024-11-18T23:15:42,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/e5e3bf6598154603becb0e42be0de2a0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/e5e3bf6598154603becb0e42be0de2a0 2024-11-18T23:15:42,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/06ee867e65f04749943c1f2f21542451 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/06ee867e65f04749943c1f2f21542451 2024-11-18T23:15:42,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3c2c6540d04f4068bcd68c91b284feb2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3c2c6540d04f4068bcd68c91b284feb2 2024-11-18T23:15:42,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5f6dd9bfe0c34b9eb8078b0ebb8891d6 2024-11-18T23:15:42,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/349b88ab90de44b08883a45e003e8a32 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/349b88ab90de44b08883a45e003e8a32 2024-11-18T23:15:42,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d8da56a8f3ab4437a4948d4a00171c32 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/d8da56a8f3ab4437a4948d4a00171c32 2024-11-18T23:15:42,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4357ef97862843b0a7f8d93a88ce67c0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4357ef97862843b0a7f8d93a88ce67c0 2024-11-18T23:15:42,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/fd46b45aa4974d289b6618cee25c87c4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/fd46b45aa4974d289b6618cee25c87c4 2024-11-18T23:15:42,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/f455ff19147e4627b7a210597680354d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/f455ff19147e4627b7a210597680354d 2024-11-18T23:15:42,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4e9f6621b02a4124a77e71751567888d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/4e9f6621b02a4124a77e71751567888d 2024-11-18T23:15:42,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5617401f9b5f457c9d6cb7828b7904e5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/5617401f9b5f457c9d6cb7828b7904e5 2024-11-18T23:15:42,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/35f7bd1096fb416190ad932a204f6157 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/35f7bd1096fb416190ad932a204f6157 2024-11-18T23:15:42,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/44d3c9169fba4edeb1bb10b2d92e619d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/44d3c9169fba4edeb1bb10b2d92e619d 2024-11-18T23:15:42,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9b3b8c9baacb4f7986e4d4a5dcf107d3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9b3b8c9baacb4f7986e4d4a5dcf107d3 2024-11-18T23:15:42,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/79f7621c7bbb475996ff48c44b3d8b9d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/79f7621c7bbb475996ff48c44b3d8b9d 2024-11-18T23:15:42,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/867a2da11ab44bc8a7f173bb5e21e9e2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/867a2da11ab44bc8a7f173bb5e21e9e2 2024-11-18T23:15:42,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3208338eb7034138be947f6dcf68b670 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/3208338eb7034138be947f6dcf68b670 2024-11-18T23:15:42,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/41bc96205336460588d94754d1056574 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/41bc96205336460588d94754d1056574 2024-11-18T23:15:42,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/6b1def59da5544948627138bf1f90826 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/6b1def59da5544948627138bf1f90826 2024-11-18T23:15:42,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/8da2312ba6f94cf89dad87df76e95aa0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/8da2312ba6f94cf89dad87df76e95aa0 2024-11-18T23:15:42,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/67e92f8a05694bbfb5942d01a624e856 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/67e92f8a05694bbfb5942d01a624e856 2024-11-18T23:15:42,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9a2614c11569478f88add7ddb789032a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/9a2614c11569478f88add7ddb789032a 2024-11-18T23:15:42,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/0c43cbeea1a64616b9207e201e02851f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/0c43cbeea1a64616b9207e201e02851f 2024-11-18T23:15:42,785 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/recovered.edits/534.seqid, newMaxSeqId=534, maxSeqId=1 2024-11-18T23:15:42,786 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. 2024-11-18T23:15:42,786 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] regionserver.HRegion(1676): Region close journal for 14ab88ca9857fdcfce5643ad2ad0a965: Waiting for close lock at 1731971740738Running coprocessor pre-close hooks at 1731971740738Disabling compacts and flushes for region at 1731971740738Disabling writes for close at 1731971741205 (+467 ms)Obtaining lock to block concurrent updates at 1731971741206 (+1 ms)Preparing flush snapshotting stores in 14ab88ca9857fdcfce5643ad2ad0a965 at 1731971741206Finished memstore snapshotting TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731971741206Flushing stores of TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965. at 1731971741207 (+1 ms)Flushing 14ab88ca9857fdcfce5643ad2ad0a965/A: creating writer at 1731971741207Flushing 14ab88ca9857fdcfce5643ad2ad0a965/A: appending metadata at 1731971741211 (+4 ms)Flushing 14ab88ca9857fdcfce5643ad2ad0a965/A: closing flushed file at 1731971741211Flushing 14ab88ca9857fdcfce5643ad2ad0a965/B: creating writer at 1731971741628 (+417 ms)Flushing 14ab88ca9857fdcfce5643ad2ad0a965/B: appending metadata at 1731971741634 (+6 ms)Flushing 14ab88ca9857fdcfce5643ad2ad0a965/B: closing flushed file at 1731971741634Flushing 14ab88ca9857fdcfce5643ad2ad0a965/C: creating writer at 1731971742060 (+426 ms)Flushing 14ab88ca9857fdcfce5643ad2ad0a965/C: appending metadata at 1731971742064 (+4 ms)Flushing 14ab88ca9857fdcfce5643ad2ad0a965/C: closing flushed file at 1731971742064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d993145: reopening flushed file at 1731971742476 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ff21111: reopening flushed file at 1731971742483 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6aaa610f: reopening flushed file at 1731971742491 (+8 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 14ab88ca9857fdcfce5643ad2ad0a965 in 1294ms, sequenceid=531, compaction requested=true at 1731971742499 (+8 ms)Writing region close event to WAL at 1731971742776 (+277 ms)Running coprocessor post-close hooks at 1731971742786 (+10 ms)Closed at 1731971742786 2024-11-18T23:15:42,789 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=26}] handler.UnassignRegionHandler(157): Closed 14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:42,790 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=14ab88ca9857fdcfce5643ad2ad0a965, regionState=CLOSED 2024-11-18T23:15:42,792 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; CloseRegionProcedure 14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:15:42,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-18T23:15:42,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseRegionProcedure 14ab88ca9857fdcfce5643ad2ad0a965, server=9d96596e55a9,32899,1731971713394 in 2.2230 sec 2024-11-18T23:15:42,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-18T23:15:42,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=14ab88ca9857fdcfce5643ad2ad0a965, UNASSIGN in 2.2320 sec 2024-11-18T23:15:42,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-18T23:15:42,802 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2420 sec 2024-11-18T23:15:42,804 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971742803"}]},"ts":"1731971742803"} 2024-11-18T23:15:42,805 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T23:15:42,806 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T23:15:42,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2620 sec 2024-11-18T23:15:42,928 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T23:15:44,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-18T23:15:44,728 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:44,733 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T23:15:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:44,741 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=27, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:44,743 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=27, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:44,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T23:15:44,750 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:44,756 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/recovered.edits] 2024-11-18T23:15:44,763 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/3badfcab0e484b1da02fd6104b3fcd5a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/3badfcab0e484b1da02fd6104b3fcd5a 2024-11-18T23:15:44,765 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/eef46b094a9c4035bbd0c6db2d35b9c9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/A/eef46b094a9c4035bbd0c6db2d35b9c9 2024-11-18T23:15:44,797 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3091d6c14315491fb073892af9392296 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/3091d6c14315491fb073892af9392296 2024-11-18T23:15:44,810 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5e7e8d5a495f4d6c952b3a08205eed23 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/B/5e7e8d5a495f4d6c952b3a08205eed23 2024-11-18T23:15:44,839 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/211070b9abed48acb2cbd5e035ea2c4f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/211070b9abed48acb2cbd5e035ea2c4f 2024-11-18T23:15:44,843 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/2283067472ce495488f7bbaa4aaa5336 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/2283067472ce495488f7bbaa4aaa5336 2024-11-18T23:15:44,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T23:15:44,851 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/25b8ea349de844278753035c9a9f4230 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/25b8ea349de844278753035c9a9f4230 2024-11-18T23:15:44,858 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/7b7096488d0f4d6ea0c4680618f6b66d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/C/7b7096488d0f4d6ea0c4680618f6b66d 2024-11-18T23:15:44,866 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/recovered.edits/534.seqid to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965/recovered.edits/534.seqid 2024-11-18T23:15:44,867 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/14ab88ca9857fdcfce5643ad2ad0a965 2024-11-18T23:15:44,868 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T23:15:44,892 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=27, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:44,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-18T23:15:44,909 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T23:15:44,977 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T23:15:44,988 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=27, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:44,988 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T23:15:44,989 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731971744988"}]},"ts":"9223372036854775807"} 2024-11-18T23:15:45,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T23:15:45,010 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 14ab88ca9857fdcfce5643ad2ad0a965, NAME => 'TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T23:15:45,011 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T23:15:45,011 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731971745011"}]},"ts":"9223372036854775807"} 2024-11-18T23:15:45,032 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T23:15:45,040 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=27, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:45,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 306 msec 2024-11-18T23:15:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-18T23:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T23:15:45,062 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:45,089 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674420971_22 at /127.0.0.1:42080 [Waiting for operation #230] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1122032795_22 at /127.0.0.1:41946 [Waiting for operation #246] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674420971_22 at /127.0.0.1:52842 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9d96596e55a9:32899-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1122032795_22 at /127.0.0.1:38542 [Waiting for operation #128] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1021 (was 698) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6133 (was 5975) - AvailableMemoryMB LEAK? - 2024-11-18T23:15:45,113 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=1021, ProcessCount=11, AvailableMemoryMB=6130 2024-11-18T23:15:45,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:15:45,116 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:15:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:45,128 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:15:45,128 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:45,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 28 2024-11-18T23:15:45,130 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:15:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T23:15:45,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741935_1111 (size=1061) 2024-11-18T23:15:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T23:15:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T23:15:45,570 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:15:45,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741936_1112 (size=53) 2024-11-18T23:15:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T23:15:45,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:45,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2312cbe93ba4cc288efe5635ed9f3f01, disabling compactions & flushes 2024-11-18T23:15:45,985 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:45,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:45,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. after waiting 0 ms 2024-11-18T23:15:45,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:45,985 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:45,985 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2312cbe93ba4cc288efe5635ed9f3f01: Waiting for close lock at 1731971745985Disabling compacts and flushes for region at 1731971745985Disabling writes for close at 1731971745985Writing region close event to WAL at 1731971745985Closed at 1731971745985 2024-11-18T23:15:45,987 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:15:45,987 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731971745987"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731971745987"}]},"ts":"1731971745987"} 2024-11-18T23:15:45,990 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:15:45,992 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:15:45,992 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971745992"}]},"ts":"1731971745992"} 2024-11-18T23:15:45,994 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T23:15:45,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, ASSIGN}] 2024-11-18T23:15:45,996 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, ASSIGN 2024-11-18T23:15:45,997 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, ASSIGN; state=OFFLINE, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=false 2024-11-18T23:15:46,147 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:46,153 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, ASSIGN because future has completed 2024-11-18T23:15:46,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:15:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T23:15:46,319 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:46,319 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7752): Opening region: {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:15:46,320 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,320 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:46,320 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7794): checking encryption for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,320 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(7797): checking classloading for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,324 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,329 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:46,330 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2312cbe93ba4cc288efe5635ed9f3f01 columnFamilyName A 2024-11-18T23:15:46,330 DEBUG [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:46,330 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(327): Store=2312cbe93ba4cc288efe5635ed9f3f01/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:46,331 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,332 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:46,333 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2312cbe93ba4cc288efe5635ed9f3f01 columnFamilyName B 2024-11-18T23:15:46,333 DEBUG [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:46,334 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(327): Store=2312cbe93ba4cc288efe5635ed9f3f01/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:46,334 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,338 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:46,338 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2312cbe93ba4cc288efe5635ed9f3f01 columnFamilyName C 2024-11-18T23:15:46,339 DEBUG [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:46,339 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(327): Store=2312cbe93ba4cc288efe5635ed9f3f01/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:46,340 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:46,340 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1038): replaying wal for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,343 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,344 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,353 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1048): stopping wal replay for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,353 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1060): Cleaning up temporary data for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,356 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:15:46,361 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1093): writing seq id for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,370 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:15:46,372 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1114): Opened 2312cbe93ba4cc288efe5635ed9f3f01; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66262611, jitterRate=-0.01261015236377716}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:15:46,372 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:46,389 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegion(1006): Region open journal for 2312cbe93ba4cc288efe5635ed9f3f01: Running coprocessor pre-open hook at 1731971746320Writing region info on filesystem at 1731971746320Initializing all the Stores at 1731971746322 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971746322Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971746324 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971746324Cleaning up temporary data from old regions at 1731971746353 (+29 ms)Running coprocessor post-open hooks at 1731971746373 (+20 ms)Region opened successfully at 1731971746388 (+15 ms) 2024-11-18T23:15:46,390 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., pid=30, masterSystemTime=1731971746306 2024-11-18T23:15:46,393 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:46,393 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=30}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:46,394 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:46,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:15:46,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-18T23:15:46,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 in 278 msec 2024-11-18T23:15:46,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-18T23:15:46,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, ASSIGN in 454 msec 2024-11-18T23:15:46,462 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:15:46,462 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971746462"}]},"ts":"1731971746462"} 2024-11-18T23:15:46,465 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T23:15:46,470 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=28, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:15:46,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3540 sec 2024-11-18T23:15:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=28 2024-11-18T23:15:47,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:47,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bcb2494, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:47,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:47,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:47,281 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ddb9398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:47,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:47,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:47,284 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:47,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641dcc92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:47,286 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:47,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:47,290 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:47,293 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:15:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,42339,1731971712494 2024-11-18T23:15:47,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@296fe578 2024-11-18T23:15:47,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:15:47,297 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:15:47,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:15:47,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:15:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T23:15:47,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741937_1113 (size=1097) 2024-11-18T23:15:47,754 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T23:15:47,754 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T23:15:47,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:15:47,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, REOPEN/MOVE}] 2024-11-18T23:15:47,770 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, REOPEN/MOVE 2024-11-18T23:15:47,771 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:47,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, REOPEN/MOVE because future has completed 2024-11-18T23:15:47,774 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:15:47,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:15:47,930 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(122): Close 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:47,930 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:15:47,930 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1722): Closing 2312cbe93ba4cc288efe5635ed9f3f01, disabling compactions & flushes 2024-11-18T23:15:47,930 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:47,930 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:47,930 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. after waiting 0 ms 2024-11-18T23:15:47,930 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:47,949 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T23:15:47,950 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:47,950 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegion(1676): Region close journal for 2312cbe93ba4cc288efe5635ed9f3f01: Waiting for close lock at 1731971747930Running coprocessor pre-close hooks at 1731971747930Disabling compacts and flushes for region at 1731971747930Disabling writes for close at 1731971747930Writing region close event to WAL at 1731971747936 (+6 ms)Running coprocessor post-close hooks at 1731971747950 (+14 ms)Closed at 1731971747950 2024-11-18T23:15:47,950 WARN [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] regionserver.HRegionServer(3299): Not adding moved region record: 2312cbe93ba4cc288efe5635ed9f3f01 to self. 2024-11-18T23:15:47,954 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=34}] handler.UnassignRegionHandler(157): Closed 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:47,956 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=CLOSED 2024-11-18T23:15:47,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:15:47,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-18T23:15:47,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 in 209 msec 2024-11-18T23:15:47,988 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, REOPEN/MOVE; state=CLOSED, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=true 2024-11-18T23:15:48,139 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:48,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, REOPEN/MOVE because future has completed 2024-11-18T23:15:48,142 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:15:48,300 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,300 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:15:48,301 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,301 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:15:48,301 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,301 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,308 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,309 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:48,315 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2312cbe93ba4cc288efe5635ed9f3f01 columnFamilyName A 2024-11-18T23:15:48,319 DEBUG [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:48,320 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(327): Store=2312cbe93ba4cc288efe5635ed9f3f01/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:48,321 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,322 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:48,322 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2312cbe93ba4cc288efe5635ed9f3f01 columnFamilyName B 2024-11-18T23:15:48,322 DEBUG [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:48,323 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(327): Store=2312cbe93ba4cc288efe5635ed9f3f01/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:48,323 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,324 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:15:48,324 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2312cbe93ba4cc288efe5635ed9f3f01 columnFamilyName C 2024-11-18T23:15:48,324 DEBUG [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:48,325 INFO [StoreOpener-2312cbe93ba4cc288efe5635ed9f3f01-1 {}] regionserver.HStore(327): Store=2312cbe93ba4cc288efe5635ed9f3f01/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:15:48,325 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,325 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,326 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,327 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,329 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,329 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,329 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:15:48,331 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,332 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 2312cbe93ba4cc288efe5635ed9f3f01; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66512836, jitterRate=-0.00888150930404663}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:15:48,332 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,333 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 2312cbe93ba4cc288efe5635ed9f3f01: Running coprocessor pre-open hook at 1731971748301Writing region info on filesystem at 1731971748301Initializing all the Stores at 1731971748302 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971748302Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971748305 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971748305Cleaning up temporary data from old regions at 1731971748329 (+24 ms)Running coprocessor post-open hooks at 1731971748332 (+3 ms)Region opened successfully at 1731971748333 (+1 ms) 2024-11-18T23:15:48,334 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., pid=35, masterSystemTime=1731971748296 2024-11-18T23:15:48,335 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:15:48,339 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=OPEN, openSeqNum=5, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:48,341 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,341 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:15:48,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=33 2024-11-18T23:15:48,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 in 201 msec 2024-11-18T23:15:48,354 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-18T23:15:48,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 592 msec 2024-11-18T23:15:48,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-18T23:15:48,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, REOPEN/MOVE in 578 msec 2024-11-18T23:15:48,361 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0500 sec 2024-11-18T23:15:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-18T23:15:48,372 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:48,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,379 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7273aef9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,382 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43b9f6e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,392 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,392 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,393 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,394 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad43a4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,397 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38546, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67577c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,407 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c93dd31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,410 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66fd648c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,426 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a4089ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,431 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54da18c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26dcfede, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,450 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4268c4cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@170b1231, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,456 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b54040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aa906a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,474 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f9067a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,477 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b0905f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,484 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,486 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2365e90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,493 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,494 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f1458f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ade04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:15:48,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:15:48,497 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:15:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:15:48,497 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:15:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc3529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:15:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:15:48,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:15:48,500 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:15:48,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442cd356, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:15:48,513 DEBUG [Thread-586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,533 DEBUG [Thread-592 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,544 DEBUG [Thread-590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,546 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,550 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,553 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,556 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,572 DEBUG [Thread-597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,572 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-18T23:15:48,577 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,578 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T23:15:48,580 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,581 DEBUG [Thread-588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,582 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:48,582 DEBUG [Thread-588 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,582 DEBUG [Thread-588 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,584 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,588 DEBUG [Thread-594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:15:48,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:15:48,590 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:48,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:48,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:15:48,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:15:48,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:15:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:48,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:48,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T23:15:48,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:48,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:48,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:48,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118401e2122859d49ed9fb26ba31ee10a3c_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971748655/Put/seqid=0 2024-11-18T23:15:48,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971748741, max timestamp=1731971748786Num uniques 859; 2024-11-18T23:15:48,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:48,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971748741, max timestamp=1731971748786Num uniques 859; 2024-11-18T23:15:48,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:48,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971748741, max timestamp=1731971748786Num uniques 859; 2024-11-18T23:15:48,804 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:48,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741938_1114 (size=12154) 2024-11-18T23:15:48,837 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:48,866 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118401e2122859d49ed9fb26ba31ee10a3c_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118401e2122859d49ed9fb26ba31ee10a3c_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:48,874 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d3601f943fee440dac7a7e51c0ecc8bc, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:48,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d3601f943fee440dac7a7e51c0ecc8bc is 175, key is test_row_0/A:col10/1731971748655/Put/seqid=0 2024-11-18T23:15:48,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T23:15:48,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:48,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:48,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:48,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741939_1115 (size=30955) 2024-11-18T23:15:48,939 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d3601f943fee440dac7a7e51c0ecc8bc 2024-11-18T23:15:48,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:48,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971808909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:48,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:48,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971808916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:48,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:48,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971808923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:48,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:48,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971808926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:48,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:48,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971808932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:48,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/326dcbbbe28947b2bc7d7e939b07cd77 is 50, key is test_row_0/B:col10/1731971748655/Put/seqid=0 2024-11-18T23:15:49,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971809048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971809070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971809070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971809072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971809073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741940_1116 (size=12001) 2024-11-18T23:15:49,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T23:15:49,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971809260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971809278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971809284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,295 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971809288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971809288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:49,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:49,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/326dcbbbe28947b2bc7d7e939b07cd77 2024-11-18T23:15:49,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971809577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/cfc58943861b46caa35f0c2108e15c80 is 50, key is test_row_0/C:col10/1731971748655/Put/seqid=0 2024-11-18T23:15:49,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971809600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971809608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971809608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971809618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:49,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741941_1117 (size=12001) 2024-11-18T23:15:49,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/cfc58943861b46caa35f0c2108e15c80 2024-11-18T23:15:49,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d3601f943fee440dac7a7e51c0ecc8bc as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d3601f943fee440dac7a7e51c0ecc8bc 2024-11-18T23:15:49,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d3601f943fee440dac7a7e51c0ecc8bc, entries=150, sequenceid=14, filesize=30.2 K 2024-11-18T23:15:49,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/326dcbbbe28947b2bc7d7e939b07cd77 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/326dcbbbe28947b2bc7d7e939b07cd77 2024-11-18T23:15:49,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/326dcbbbe28947b2bc7d7e939b07cd77, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T23:15:49,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/cfc58943861b46caa35f0c2108e15c80 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/cfc58943861b46caa35f0c2108e15c80 2024-11-18T23:15:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T23:15:49,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:49,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/cfc58943861b46caa35f0c2108e15c80, entries=150, sequenceid=14, filesize=11.7 K 2024-11-18T23:15:49,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1068ms, sequenceid=14, compaction requested=false 2024-11-18T23:15:49,745 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T23:15:49,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:49,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-18T23:15:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:49,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T23:15:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:49,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:49,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184671c78c50c14d9fa3ec8618e826bc15_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971748926/Put/seqid=0 2024-11-18T23:15:49,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741942_1118 (size=12154) 2024-11-18T23:15:49,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:49,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184671c78c50c14d9fa3ec8618e826bc15_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184671c78c50c14d9fa3ec8618e826bc15_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:49,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6ff405017aed4e3db2e4ad0e0c617366, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:49,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6ff405017aed4e3db2e4ad0e0c617366 is 175, key is test_row_0/A:col10/1731971748926/Put/seqid=0 2024-11-18T23:15:50,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741943_1119 (size=30955) 2024-11-18T23:15:50,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6ff405017aed4e3db2e4ad0e0c617366 2024-11-18T23:15:50,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/f0795523c19e4f24be76977a2c9ca667 is 50, key is test_row_0/B:col10/1731971748926/Put/seqid=0 2024-11-18T23:15:50,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741944_1120 (size=12001) 2024-11-18T23:15:50,066 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/f0795523c19e4f24be76977a2c9ca667 2024-11-18T23:15:50,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/d79ba7f95ee3478e87e9d00be5309b2a is 50, key is test_row_0/C:col10/1731971748926/Put/seqid=0 2024-11-18T23:15:50,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971810132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971810131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971810133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971810135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971810144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741945_1121 (size=12001) 2024-11-18T23:15:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,169 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/d79ba7f95ee3478e87e9d00be5309b2a 2024-11-18T23:15:50,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6ff405017aed4e3db2e4ad0e0c617366 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6ff405017aed4e3db2e4ad0e0c617366 2024-11-18T23:15:50,232 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6ff405017aed4e3db2e4ad0e0c617366, entries=150, sequenceid=51, filesize=30.2 K 2024-11-18T23:15:50,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/f0795523c19e4f24be76977a2c9ca667 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f0795523c19e4f24be76977a2c9ca667 2024-11-18T23:15:50,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971810248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,250 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971810248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,265 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f0795523c19e4f24be76977a2c9ca667, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T23:15:50,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/d79ba7f95ee3478e87e9d00be5309b2a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d79ba7f95ee3478e87e9d00be5309b2a 2024-11-18T23:15:50,296 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d79ba7f95ee3478e87e9d00be5309b2a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-18T23:15:50,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 2312cbe93ba4cc288efe5635ed9f3f01 in 403ms, sequenceid=51, compaction requested=false 2024-11-18T23:15:50,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:50,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:50,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-18T23:15:50,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-18T23:15:50,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-18T23:15:50,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7110 sec 2024-11-18T23:15:50,316 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.7400 sec 2024-11-18T23:15:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:50,469 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:15:50,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:50,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:50,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:50,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:50,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:50,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:50,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185ccfc7fd74cd4126bc6fdbc870bb6313_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971750126/Put/seqid=0 2024-11-18T23:15:50,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741946_1122 (size=14594) 2024-11-18T23:15:50,526 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:50,542 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185ccfc7fd74cd4126bc6fdbc870bb6313_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185ccfc7fd74cd4126bc6fdbc870bb6313_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:50,544 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6a609a0e3b1e417bb4aa0c7df0e41af9, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:50,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6a609a0e3b1e417bb4aa0c7df0e41af9 is 175, key is test_row_0/A:col10/1731971750126/Put/seqid=0 2024-11-18T23:15:50,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741947_1123 (size=39549) 2024-11-18T23:15:50,558 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6a609a0e3b1e417bb4aa0c7df0e41af9 2024-11-18T23:15:50,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731971750533, max timestamp=1731971750570Num uniques 859; 2024-11-18T23:15:50,573 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T23:15:50,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971750541, max timestamp=1731971750570Num uniques 859; 2024-11-18T23:15:50,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971750541, max timestamp=1731971750570Num uniques 859; 2024-11-18T23:15:50,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:50,576 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:50,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/c25d7a40b3aa4e0ebb989608158fd99c is 50, key is test_row_0/B:col10/1731971750126/Put/seqid=0 2024-11-18T23:15:50,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741948_1124 (size=12001) 2024-11-18T23:15:50,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/c25d7a40b3aa4e0ebb989608158fd99c 2024-11-18T23:15:50,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/701f86e608ef47518cc75e0f399d5f45 is 50, key is test_row_0/C:col10/1731971750126/Put/seqid=0 2024-11-18T23:15:50,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971810659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971810661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741949_1125 (size=12001) 2024-11-18T23:15:50,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/701f86e608ef47518cc75e0f399d5f45 2024-11-18T23:15:50,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/6a609a0e3b1e417bb4aa0c7df0e41af9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6a609a0e3b1e417bb4aa0c7df0e41af9 2024-11-18T23:15:50,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6a609a0e3b1e417bb4aa0c7df0e41af9, entries=200, sequenceid=62, filesize=38.6 K 2024-11-18T23:15:50,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/c25d7a40b3aa4e0ebb989608158fd99c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c25d7a40b3aa4e0ebb989608158fd99c 2024-11-18T23:15:50,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-18T23:15:50,751 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:50,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c25d7a40b3aa4e0ebb989608158fd99c, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T23:15:50,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-18T23:15:50,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T23:15:50,760 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:50,761 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:50,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:50,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/701f86e608ef47518cc75e0f399d5f45 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/701f86e608ef47518cc75e0f399d5f45 2024-11-18T23:15:50,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/701f86e608ef47518cc75e0f399d5f45, entries=150, sequenceid=62, filesize=11.7 K 2024-11-18T23:15:50,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2312cbe93ba4cc288efe5635ed9f3f01 in 309ms, sequenceid=62, compaction requested=true 2024-11-18T23:15:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:50,779 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:50,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:50,779 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:50,781 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:50,781 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/B is initiating minor compaction (all files) 2024-11-18T23:15:50,781 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:50,781 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/B in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:50,781 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/A is initiating minor compaction (all files) 2024-11-18T23:15:50,781 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/A in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:50,781 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/326dcbbbe28947b2bc7d7e939b07cd77, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f0795523c19e4f24be76977a2c9ca667, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c25d7a40b3aa4e0ebb989608158fd99c] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=35.2 K 2024-11-18T23:15:50,781 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d3601f943fee440dac7a7e51c0ecc8bc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6ff405017aed4e3db2e4ad0e0c617366, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6a609a0e3b1e417bb4aa0c7df0e41af9] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=99.1 K 2024-11-18T23:15:50,781 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:50,782 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d3601f943fee440dac7a7e51c0ecc8bc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6ff405017aed4e3db2e4ad0e0c617366, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6a609a0e3b1e417bb4aa0c7df0e41af9] 2024-11-18T23:15:50,782 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 326dcbbbe28947b2bc7d7e939b07cd77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731971748655 2024-11-18T23:15:50,782 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d3601f943fee440dac7a7e51c0ecc8bc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731971748655 2024-11-18T23:15:50,783 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0795523c19e4f24be76977a2c9ca667, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731971748906 2024-11-18T23:15:50,783 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ff405017aed4e3db2e4ad0e0c617366, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731971748906 2024-11-18T23:15:50,783 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting c25d7a40b3aa4e0ebb989608158fd99c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731971750126 2024-11-18T23:15:50,784 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a609a0e3b1e417bb4aa0c7df0e41af9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731971750095 2024-11-18T23:15:50,805 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:50,809 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T23:15:50,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:50,812 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:50,813 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/4ae967cdcd57442fa4482c2438115311 is 50, key is test_row_0/B:col10/1731971750126/Put/seqid=0 2024-11-18T23:15:50,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731971750654, max timestamp=1731971750804Num uniques 1018; 2024-11-18T23:15:50,816 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-18T23:15:50,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:50,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:15:50,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:50,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:50,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:50,819 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:15:50,833 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118f2631a1815ba49479686763c49769795_2312cbe93ba4cc288efe5635ed9f3f01 store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:50,838 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118f2631a1815ba49479686763c49769795_2312cbe93ba4cc288efe5635ed9f3f01, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:50,838 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f2631a1815ba49479686763c49769795_2312cbe93ba4cc288efe5635ed9f3f01 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:50,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T23:15:50,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111828e142d3c77b435e86715855666ff34b_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971750654/Put/seqid=0 2024-11-18T23:15:50,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741950_1126 (size=12104) 2024-11-18T23:15:50,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741951_1127 (size=4469) 2024-11-18T23:15:50,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:50,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:50,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:50,919 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#A#compaction#107 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:50,920 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/2933ba0e2b2c47fea4962f71f3622686 is 175, key is test_row_0/A:col10/1731971750126/Put/seqid=0 2024-11-18T23:15:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971810915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971810930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:50,947 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/4ae967cdcd57442fa4482c2438115311 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/4ae967cdcd57442fa4482c2438115311 2024-11-18T23:15:50,984 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/B of 2312cbe93ba4cc288efe5635ed9f3f01 into 4ae967cdcd57442fa4482c2438115311(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:50,984 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:50,984 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/B, priority=13, startTime=1731971750779; duration=0sec 2024-11-18T23:15:50,985 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:50,985 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:B 2024-11-18T23:15:50,985 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:15:50,989 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:15:50,989 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/C is initiating minor compaction (all files) 2024-11-18T23:15:50,989 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/C in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:50,990 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/cfc58943861b46caa35f0c2108e15c80, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d79ba7f95ee3478e87e9d00be5309b2a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/701f86e608ef47518cc75e0f399d5f45] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=35.2 K 2024-11-18T23:15:50,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741952_1128 (size=14594) 2024-11-18T23:15:50,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741953_1129 (size=31058) 2024-11-18T23:15:50,997 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfc58943861b46caa35f0c2108e15c80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731971748655 2024-11-18T23:15:50,998 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d79ba7f95ee3478e87e9d00be5309b2a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731971748906 2024-11-18T23:15:50,999 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:51,012 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 701f86e608ef47518cc75e0f399d5f45, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731971750126 2024-11-18T23:15:51,032 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111828e142d3c77b435e86715855666ff34b_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828e142d3c77b435e86715855666ff34b_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:51,056 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/2933ba0e2b2c47fea4962f71f3622686 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2933ba0e2b2c47fea4962f71f3622686 2024-11-18T23:15:51,057 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d0bed6f7af0a46d2bbb643c2c6631589, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:51,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d0bed6f7af0a46d2bbb643c2c6631589 is 175, key is test_row_0/A:col10/1731971750654/Put/seqid=0 2024-11-18T23:15:51,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971811037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971811048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,074 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T23:15:51,100 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/A of 2312cbe93ba4cc288efe5635ed9f3f01 into 2933ba0e2b2c47fea4962f71f3622686(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:51,100 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:51,101 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/A, priority=13, startTime=1731971750779; duration=0sec 2024-11-18T23:15:51,102 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:51,102 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:A 2024-11-18T23:15:51,106 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#C#compaction#110 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:51,107 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/a015805562ce4e2aa775117536e72864 is 50, key is test_row_0/C:col10/1731971750126/Put/seqid=0 2024-11-18T23:15:51,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741954_1130 (size=39549) 2024-11-18T23:15:51,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741955_1131 (size=12104) 2024-11-18T23:15:51,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971811148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,154 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 2312cbe93ba4cc288efe5635ed9f3f01 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:15:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,159 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/a015805562ce4e2aa775117536e72864 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a015805562ce4e2aa775117536e72864 2024-11-18T23:15:51,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971811163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,172 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_0' in 2312cbe93ba4cc288efe5635ed9f3f01 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:15:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,176 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/C of 2312cbe93ba4cc288efe5635ed9f3f01 into a015805562ce4e2aa775117536e72864(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:51,176 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:51,176 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/C, priority=13, startTime=1731971750779; duration=0sec 2024-11-18T23:15:51,178 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:51,178 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:C 2024-11-18T23:15:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971811178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,181 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 2312cbe93ba4cc288efe5635ed9f3f01 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971811278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971811278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,282 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:51,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:51,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T23:15:51,517 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d0bed6f7af0a46d2bbb643c2c6631589 2024-11-18T23:15:51,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/50968f5306bc45f98363903d35f0bbe5 is 50, key is test_row_0/B:col10/1731971750654/Put/seqid=0 2024-11-18T23:15:51,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971811589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971811592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:51,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741956_1132 (size=12001) 2024-11-18T23:15:51,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:51,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:51,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:51,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T23:15:52,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:52,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:52,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:52,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:52,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/50968f5306bc45f98363903d35f0bbe5 2024-11-18T23:15:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:52,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/29c84aa0ec674a9c8bf16fff31e2c49d is 50, key is test_row_0/C:col10/1731971750654/Put/seqid=0 2024-11-18T23:15:52,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741957_1133 (size=12001) 2024-11-18T23:15:52,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/29c84aa0ec674a9c8bf16fff31e2c49d 2024-11-18T23:15:52,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/d0bed6f7af0a46d2bbb643c2c6631589 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d0bed6f7af0a46d2bbb643c2c6631589 2024-11-18T23:15:52,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d0bed6f7af0a46d2bbb643c2c6631589, entries=200, sequenceid=100, filesize=38.6 K 2024-11-18T23:15:52,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/50968f5306bc45f98363903d35f0bbe5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/50968f5306bc45f98363903d35f0bbe5 2024-11-18T23:15:52,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971812108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:52,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971812109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:52,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:52,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:52,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:52,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/50968f5306bc45f98363903d35f0bbe5, entries=150, sequenceid=100, filesize=11.7 K 2024-11-18T23:15:52,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/29c84aa0ec674a9c8bf16fff31e2c49d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/29c84aa0ec674a9c8bf16fff31e2c49d 2024-11-18T23:15:52,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/29c84aa0ec674a9c8bf16fff31e2c49d, entries=150, sequenceid=100, filesize=11.7 K 2024-11-18T23:15:52,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~111.82 KB/114500, heapSize ~312.47 KB/319968, currentSize=67.09 KB/68700 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1347ms, sequenceid=100, compaction requested=false 2024-11-18T23:15:52,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:52,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-18T23:15:52,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:52,174 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:15:52,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:52,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:52,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:52,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:52,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:52,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118533edafbf5a64c5d8c77f3ef6652527e_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971750897/Put/seqid=0 2024-11-18T23:15:52,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741958_1134 (size=12154) 2024-11-18T23:15:52,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:52,646 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118533edafbf5a64c5d8c77f3ef6652527e_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118533edafbf5a64c5d8c77f3ef6652527e_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:52,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/e618345965e64814ae5ec3b4033fdb3d, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:52,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/e618345965e64814ae5ec3b4033fdb3d is 175, key is test_row_0/A:col10/1731971750897/Put/seqid=0 2024-11-18T23:15:52,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741959_1135 (size=30955) 2024-11-18T23:15:52,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T23:15:52,927 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T23:15:52,928 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T23:15:53,115 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/e618345965e64814ae5ec3b4033fdb3d 2024-11-18T23:15:53,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:53,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:53,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/13755b8023b744e1ab52e7f60901788f is 50, key is test_row_0/B:col10/1731971750897/Put/seqid=0 2024-11-18T23:15:53,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741960_1136 (size=12001) 2024-11-18T23:15:53,217 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/13755b8023b744e1ab52e7f60901788f 2024-11-18T23:15:53,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/2ac8b24bd08c46e28346907ca6da4aa1 is 50, key is test_row_0/C:col10/1731971750897/Put/seqid=0 2024-11-18T23:15:53,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971813241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,261 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971813246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971813257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971813265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971813269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741961_1137 (size=12001) 2024-11-18T23:15:53,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971813372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971813372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971813380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971813380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971813392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,458 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:15:53,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971813589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971813592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971813599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971813599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971813624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:53,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:53,705 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/2ac8b24bd08c46e28346907ca6da4aa1 2024-11-18T23:15:53,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/e618345965e64814ae5ec3b4033fdb3d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e618345965e64814ae5ec3b4033fdb3d 2024-11-18T23:15:53,744 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e618345965e64814ae5ec3b4033fdb3d, entries=150, sequenceid=116, filesize=30.2 K 2024-11-18T23:15:53,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/13755b8023b744e1ab52e7f60901788f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/13755b8023b744e1ab52e7f60901788f 2024-11-18T23:15:53,754 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/13755b8023b744e1ab52e7f60901788f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T23:15:53,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/2ac8b24bd08c46e28346907ca6da4aa1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/2ac8b24bd08c46e28346907ca6da4aa1 2024-11-18T23:15:53,777 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/2ac8b24bd08c46e28346907ca6da4aa1, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T23:15:53,783 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1609ms, sequenceid=116, compaction requested=true 2024-11-18T23:15:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-18T23:15:53,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-18T23:15:53,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-18T23:15:53,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0240 sec 2024-11-18T23:15:53,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 3.0330 sec 2024-11-18T23:15:53,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:53,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971753221, max timestamp=1731971753901Num uniques 859; 2024-11-18T23:15:53,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971753221, max timestamp=1731971753901Num uniques 859; 2024-11-18T23:15:53,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:53,906 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:53,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-18T23:15:53,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:53,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:53,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:53,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:53,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:53,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:53,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184c38ed91a7854aadbf18c2c171d7d674_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971753221/Put/seqid=0 2024-11-18T23:15:53,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741962_1138 (size=12304) 2024-11-18T23:15:53,989 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:53,996 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411184c38ed91a7854aadbf18c2c171d7d674_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184c38ed91a7854aadbf18c2c171d7d674_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:53,998 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/07d93b044a924a6aad079ee30fb65601, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:53,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/07d93b044a924a6aad079ee30fb65601 is 175, key is test_row_0/A:col10/1731971753221/Put/seqid=0 2024-11-18T23:15:54,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741963_1139 (size=31105) 2024-11-18T23:15:54,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971753985, max timestamp=1731971754039Num uniques 859; 2024-11-18T23:15:54,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971753985, max timestamp=1731971754039Num uniques 859; 2024-11-18T23:15:54,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:54,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:54,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971753985, max timestamp=1731971754039Num uniques 859; 2024-11-18T23:15:54,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:54,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971814117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971814117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,146 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,147 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971814127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971814128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971814124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971814261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971814261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971814261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971814270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971814270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,444 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/07d93b044a924a6aad079ee30fb65601 2024-11-18T23:15:54,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/cf8a6dcd8102434cb2bf76e386538fd4 is 50, key is test_row_0/B:col10/1731971753221/Put/seqid=0 2024-11-18T23:15:54,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971814472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971814472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971814483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971814489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971814491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741964_1140 (size=12151) 2024-11-18T23:15:54,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971814798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971814798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971814807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971814808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971814808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:54,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:54,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/cf8a6dcd8102434cb2bf76e386538fd4 2024-11-18T23:15:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-18T23:15:54,941 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:54,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:54,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-18T23:15:54,954 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:54,955 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:54,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:54,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/7fe232948289475dbd4861f1f719a95e is 50, key is test_row_0/C:col10/1731971753221/Put/seqid=0 2024-11-18T23:15:54,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T23:15:55,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741965_1141 (size=12151) 2024-11-18T23:15:55,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/7fe232948289475dbd4861f1f719a95e 2024-11-18T23:15:55,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T23:15:55,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/07d93b044a924a6aad079ee30fb65601 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/07d93b044a924a6aad079ee30fb65601 2024-11-18T23:15:55,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T23:15:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:55,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:55,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:55,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/07d93b044a924a6aad079ee30fb65601, entries=150, sequenceid=137, filesize=30.4 K 2024-11-18T23:15:55,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/cf8a6dcd8102434cb2bf76e386538fd4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cf8a6dcd8102434cb2bf76e386538fd4 2024-11-18T23:15:55,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cf8a6dcd8102434cb2bf76e386538fd4, entries=150, sequenceid=137, filesize=11.9 K 2024-11-18T23:15:55,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/7fe232948289475dbd4861f1f719a95e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/7fe232948289475dbd4861f1f719a95e 2024-11-18T23:15:55,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/7fe232948289475dbd4861f1f719a95e, entries=150, sequenceid=137, filesize=11.9 K 2024-11-18T23:15:55,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1278ms, sequenceid=137, compaction requested=true 2024-11-18T23:15:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:55,190 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:55,191 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:55,195 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132667 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:55,195 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/A is initiating minor compaction (all files) 2024-11-18T23:15:55,195 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/A in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:55,195 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2933ba0e2b2c47fea4962f71f3622686, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d0bed6f7af0a46d2bbb643c2c6631589, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e618345965e64814ae5ec3b4033fdb3d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/07d93b044a924a6aad079ee30fb65601] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=129.6 K 2024-11-18T23:15:55,195 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:55,195 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2933ba0e2b2c47fea4962f71f3622686, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d0bed6f7af0a46d2bbb643c2c6631589, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e618345965e64814ae5ec3b4033fdb3d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/07d93b044a924a6aad079ee30fb65601] 2024-11-18T23:15:55,196 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:55,196 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/B is initiating minor compaction (all files) 2024-11-18T23:15:55,196 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/B in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:55,196 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/4ae967cdcd57442fa4482c2438115311, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/50968f5306bc45f98363903d35f0bbe5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/13755b8023b744e1ab52e7f60901788f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cf8a6dcd8102434cb2bf76e386538fd4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=47.1 K 2024-11-18T23:15:55,196 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2933ba0e2b2c47fea4962f71f3622686, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731971750126 2024-11-18T23:15:55,196 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae967cdcd57442fa4482c2438115311, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731971750126 2024-11-18T23:15:55,197 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0bed6f7af0a46d2bbb643c2c6631589, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731971750654 2024-11-18T23:15:55,198 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 50968f5306bc45f98363903d35f0bbe5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731971750654 2024-11-18T23:15:55,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e618345965e64814ae5ec3b4033fdb3d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731971750897 2024-11-18T23:15:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:55,199 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 13755b8023b744e1ab52e7f60901788f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731971750897 2024-11-18T23:15:55,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:55,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d93b044a924a6aad079ee30fb65601, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731971753221 2024-11-18T23:15:55,199 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8a6dcd8102434cb2bf76e386538fd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731971753221 2024-11-18T23:15:55,221 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#B#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:55,222 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/cc65c5377ee146c192f6c63335f45115 is 50, key is test_row_0/B:col10/1731971753221/Put/seqid=0 2024-11-18T23:15:55,224 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:55,253 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118868af203bfac42eeadc43ae2ceedfdce_2312cbe93ba4cc288efe5635ed9f3f01 store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:55,259 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118868af203bfac42eeadc43ae2ceedfdce_2312cbe93ba4cc288efe5635ed9f3f01, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:55,259 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118868af203bfac42eeadc43ae2ceedfdce_2312cbe93ba4cc288efe5635ed9f3f01 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:55,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T23:15:55,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-18T23:15:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:55,288 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-18T23:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:55,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:55,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741966_1142 (size=12391) 2024-11-18T23:15:55,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741967_1143 (size=4469) 2024-11-18T23:15:55,322 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#A#compaction#120 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:55,322 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/624a5fc0023c490d9f1e9ee04180f1d8 is 175, key is test_row_0/A:col10/1731971753221/Put/seqid=0 2024-11-18T23:15:55,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182771f77b329f472fa991e13885de2101_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971754094/Put/seqid=0 2024-11-18T23:15:55,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:55,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:55,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741969_1145 (size=12304) 2024-11-18T23:15:55,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741968_1144 (size=31345) 2024-11-18T23:15:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:55,381 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182771f77b329f472fa991e13885de2101_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182771f77b329f472fa991e13885de2101_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:55,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b20f0a80f7ea4a5ba505a98bcb494fe3, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:55,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b20f0a80f7ea4a5ba505a98bcb494fe3 is 175, key is test_row_0/A:col10/1731971754094/Put/seqid=0 2024-11-18T23:15:55,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971815364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971815367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971815383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971815385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971815388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741970_1146 (size=31105) 2024-11-18T23:15:55,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971815500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971815507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971815508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971815508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971815508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T23:15:55,719 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/cc65c5377ee146c192f6c63335f45115 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cc65c5377ee146c192f6c63335f45115 2024-11-18T23:15:55,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971815708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,731 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/B of 2312cbe93ba4cc288efe5635ed9f3f01 into cc65c5377ee146c192f6c63335f45115(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:55,731 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:55,731 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/B, priority=12, startTime=1731971755190; duration=0sec 2024-11-18T23:15:55,731 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:55,731 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:B 2024-11-18T23:15:55,731 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,738 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:55,738 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/C is initiating minor compaction (all files) 2024-11-18T23:15:55,738 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/C in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:55,738 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a015805562ce4e2aa775117536e72864, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/29c84aa0ec674a9c8bf16fff31e2c49d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/2ac8b24bd08c46e28346907ca6da4aa1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/7fe232948289475dbd4861f1f719a95e] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=47.1 K 2024-11-18T23:15:55,739 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a015805562ce4e2aa775117536e72864, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731971750126 2024-11-18T23:15:55,739 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 29c84aa0ec674a9c8bf16fff31e2c49d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731971750654 2024-11-18T23:15:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971815727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,743 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ac8b24bd08c46e28346907ca6da4aa1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731971750897 2024-11-18T23:15:55,744 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fe232948289475dbd4861f1f719a95e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731971753221 2024-11-18T23:15:55,760 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#C#compaction#122 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:15:55,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,761 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/89351bbf098041b99cf50c7808e5eb64 is 50, key is test_row_0/C:col10/1731971753221/Put/seqid=0 2024-11-18T23:15:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971815738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971815738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:55,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971815738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:55,765 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:55,783 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/624a5fc0023c490d9f1e9ee04180f1d8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/624a5fc0023c490d9f1e9ee04180f1d8 2024-11-18T23:15:55,796 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/A of 2312cbe93ba4cc288efe5635ed9f3f01 into 624a5fc0023c490d9f1e9ee04180f1d8(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:55,796 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:55,796 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/A, priority=12, startTime=1731971755190; duration=0sec 2024-11-18T23:15:55,796 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:55,796 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:A 2024-11-18T23:15:55,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741971_1147 (size=12391) 2024-11-18T23:15:55,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b20f0a80f7ea4a5ba505a98bcb494fe3 2024-11-18T23:15:55,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/395af88671ae4fcb9a4872b5beb23697 is 50, key is test_row_0/B:col10/1731971754094/Put/seqid=0 2024-11-18T23:15:55,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741972_1148 (size=12151) 2024-11-18T23:15:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971816055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971816056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971816069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971816070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971816078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T23:15:56,239 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/89351bbf098041b99cf50c7808e5eb64 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/89351bbf098041b99cf50c7808e5eb64 2024-11-18T23:15:56,263 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/C of 2312cbe93ba4cc288efe5635ed9f3f01 into 89351bbf098041b99cf50c7808e5eb64(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:15:56,263 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:56,263 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/C, priority=12, startTime=1731971755190; duration=0sec 2024-11-18T23:15:56,263 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:56,263 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:C 2024-11-18T23:15:56,338 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/395af88671ae4fcb9a4872b5beb23697 2024-11-18T23:15:56,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/777043d1e5384dabb8403620e49fef88 is 50, key is test_row_0/C:col10/1731971754094/Put/seqid=0 2024-11-18T23:15:56,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741973_1149 (size=12151) 2024-11-18T23:15:56,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971816578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971816578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971816592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971816596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:56,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971816604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:56,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:56,837 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/777043d1e5384dabb8403620e49fef88 2024-11-18T23:15:56,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b20f0a80f7ea4a5ba505a98bcb494fe3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b20f0a80f7ea4a5ba505a98bcb494fe3 2024-11-18T23:15:56,903 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b20f0a80f7ea4a5ba505a98bcb494fe3, entries=150, sequenceid=173, filesize=30.4 K 2024-11-18T23:15:56,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/395af88671ae4fcb9a4872b5beb23697 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/395af88671ae4fcb9a4872b5beb23697 2024-11-18T23:15:56,946 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/395af88671ae4fcb9a4872b5beb23697, entries=150, sequenceid=173, filesize=11.9 K 2024-11-18T23:15:56,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/777043d1e5384dabb8403620e49fef88 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/777043d1e5384dabb8403620e49fef88 2024-11-18T23:15:56,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/777043d1e5384dabb8403620e49fef88, entries=150, sequenceid=173, filesize=11.9 K 2024-11-18T23:15:56,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1705ms, sequenceid=173, compaction requested=false 2024-11-18T23:15:56,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:56,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:56,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-18T23:15:57,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-18T23:15:57,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-18T23:15:57,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0520 sec 2024-11-18T23:15:57,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.0760 sec 2024-11-18T23:15:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-18T23:15:57,138 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:57,139 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-18T23:15:57,143 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:57,148 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:57,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:57,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T23:15:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T23:15:57,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-18T23:15:57,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:57,313 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:15:57,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:57,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:57,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:57,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111861c75afbc9184b328f4f748e4cf7e641_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971755386/Put/seqid=0 2024-11-18T23:15:57,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741974_1150 (size=12304) 2024-11-18T23:15:57,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:57,449 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111861c75afbc9184b328f4f748e4cf7e641_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111861c75afbc9184b328f4f748e4cf7e641_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:57,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f40cfd1e9ded4843a30946757a281b8a, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:57,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f40cfd1e9ded4843a30946757a281b8a is 175, key is test_row_0/A:col10/1731971755386/Put/seqid=0 2024-11-18T23:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T23:15:57,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741975_1151 (size=31105) 2024-11-18T23:15:57,485 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=187, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f40cfd1e9ded4843a30946757a281b8a 2024-11-18T23:15:57,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/e2edfbe40b9e463f928d3f0b17ef252b is 50, key is test_row_0/B:col10/1731971755386/Put/seqid=0 2024-11-18T23:15:57,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741976_1152 (size=12151) 2024-11-18T23:15:57,546 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/e2edfbe40b9e463f928d3f0b17ef252b 2024-11-18T23:15:57,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/505fbc76313b462bb662f6d69336fb88 is 50, key is test_row_0/C:col10/1731971755386/Put/seqid=0 2024-11-18T23:15:57,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:57,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741977_1153 (size=12151) 2024-11-18T23:15:57,644 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/505fbc76313b462bb662f6d69336fb88 2024-11-18T23:15:57,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f40cfd1e9ded4843a30946757a281b8a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f40cfd1e9ded4843a30946757a281b8a 2024-11-18T23:15:57,692 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f40cfd1e9ded4843a30946757a281b8a, entries=150, sequenceid=187, filesize=30.4 K 2024-11-18T23:15:57,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/e2edfbe40b9e463f928d3f0b17ef252b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/e2edfbe40b9e463f928d3f0b17ef252b 2024-11-18T23:15:57,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971757676, max timestamp=1731971757697Num uniques 859; 2024-11-18T23:15:57,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971757676, max timestamp=1731971757697Num uniques 859; 2024-11-18T23:15:57,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:57,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:57,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971757676, max timestamp=1731971757697Num uniques 859; 2024-11-18T23:15:57,708 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:15:57,723 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/e2edfbe40b9e463f928d3f0b17ef252b, entries=150, sequenceid=187, filesize=11.9 K 2024-11-18T23:15:57,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/505fbc76313b462bb662f6d69336fb88 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/505fbc76313b462bb662f6d69336fb88 2024-11-18T23:15:57,748 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/505fbc76313b462bb662f6d69336fb88, entries=150, sequenceid=187, filesize=11.9 K 2024-11-18T23:15:57,750 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=87.22 KB/89310 for 2312cbe93ba4cc288efe5635ed9f3f01 in 437ms, sequenceid=187, compaction requested=true 2024-11-18T23:15:57,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:57,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:57,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-18T23:15:57,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=87.22 KB heapSize=245.53 KB 2024-11-18T23:15:57,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:57,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:57,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:57,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:57,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-18T23:15:57,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:15:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:57,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-18T23:15:57,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 607 msec 2024-11-18T23:15:57,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 621 msec 2024-11-18T23:15:57,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b664bc6dd1548fb8f6eeb64313d8083_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971757709/Put/seqid=0 2024-11-18T23:15:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-18T23:15:57,792 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:15:57,796 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:15:57,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-18T23:15:57,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T23:15:57,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:15:57,800 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:15:57,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:15:57,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741978_1154 (size=12304) 2024-11-18T23:15:57,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971817828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971817830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971817838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971817848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971817849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T23:15:57,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:57,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:57,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971817958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,969 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971817961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971817964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971817969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:57,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971817981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:57,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T23:15:58,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:58,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:58,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971818183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971818183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971818183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971818183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971818201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,229 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:15:58,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411187b664bc6dd1548fb8f6eeb64313d8083_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b664bc6dd1548fb8f6eeb64313d8083_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:15:58,237 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/765bce4c322c45e49a2ed2add3c9984c, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:58,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/765bce4c322c45e49a2ed2add3c9984c is 175, key is test_row_0/A:col10/1731971757709/Put/seqid=0 2024-11-18T23:15:58,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741979_1155 (size=31105) 2024-11-18T23:15:58,359 INFO [master/9d96596e55a9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-18T23:15:58,359 INFO [master/9d96596e55a9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-18T23:15:58,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T23:15:58,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971818501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971818501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971818512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971818513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971818520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:58,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:58,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:58,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,705 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/765bce4c322c45e49a2ed2add3c9984c 2024-11-18T23:15:58,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ccbdcfed166d4503b2a8e5f31dabcf45 is 50, key is test_row_0/B:col10/1731971757709/Put/seqid=0 2024-11-18T23:15:58,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741980_1156 (size=12151) 2024-11-18T23:15:58,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:58,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:58,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T23:15:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971819026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:59,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971819026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971819032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:59,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:59,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971819039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:59,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:15:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971819052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:15:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:15:59,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:15:59,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:59,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:59,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ccbdcfed166d4503b2a8e5f31dabcf45 2024-11-18T23:15:59,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/0af57539580141c583c0d672abad7d11 is 50, key is test_row_0/C:col10/1731971757709/Put/seqid=0 2024-11-18T23:15:59,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:59,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741981_1157 (size=12151) 2024-11-18T23:15:59,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:59,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,428 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:59,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:59,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/0af57539580141c583c0d672abad7d11 2024-11-18T23:15:59,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:15:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:15:59,768 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/765bce4c322c45e49a2ed2add3c9984c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/765bce4c322c45e49a2ed2add3c9984c 2024-11-18T23:15:59,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/765bce4c322c45e49a2ed2add3c9984c, entries=150, sequenceid=217, filesize=30.4 K 2024-11-18T23:15:59,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ccbdcfed166d4503b2a8e5f31dabcf45 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ccbdcfed166d4503b2a8e5f31dabcf45 2024-11-18T23:15:59,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ccbdcfed166d4503b2a8e5f31dabcf45, entries=150, sequenceid=217, filesize=11.9 K 2024-11-18T23:15:59,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/0af57539580141c583c0d672abad7d11 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/0af57539580141c583c0d672abad7d11 2024-11-18T23:15:59,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/0af57539580141c583c0d672abad7d11, entries=150, sequenceid=217, filesize=11.9 K 2024-11-18T23:15:59,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~245.48 KB/251376, currentSize=100.63 KB/103050 for 2312cbe93ba4cc288efe5635ed9f3f01 in 2152ms, sequenceid=217, compaction requested=true 2024-11-18T23:15:59,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:15:59,905 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:15:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:15:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:15:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:15:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:15:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:15:59,907 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:59,907 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/A is initiating minor compaction (all files) 2024-11-18T23:15:59,907 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/A in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,907 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/624a5fc0023c490d9f1e9ee04180f1d8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b20f0a80f7ea4a5ba505a98bcb494fe3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f40cfd1e9ded4843a30946757a281b8a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/765bce4c322c45e49a2ed2add3c9984c] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=121.7 K 2024-11-18T23:15:59,907 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,907 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/624a5fc0023c490d9f1e9ee04180f1d8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b20f0a80f7ea4a5ba505a98bcb494fe3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f40cfd1e9ded4843a30946757a281b8a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/765bce4c322c45e49a2ed2add3c9984c] 2024-11-18T23:15:59,908 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:15:59,908 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624a5fc0023c490d9f1e9ee04180f1d8, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731971753221 2024-11-18T23:15:59,912 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b20f0a80f7ea4a5ba505a98bcb494fe3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731971754094 2024-11-18T23:15:59,913 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f40cfd1e9ded4843a30946757a281b8a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731971755378 2024-11-18T23:15:59,914 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 765bce4c322c45e49a2ed2add3c9984c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971757709 2024-11-18T23:15:59,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-18T23:15:59,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,923 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T23:15:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:15:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:15:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:15:59,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:15:59,940 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:15:59,941 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/B is initiating minor compaction (all files) 2024-11-18T23:15:59,941 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/B in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:15:59,941 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cc65c5377ee146c192f6c63335f45115, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/395af88671ae4fcb9a4872b5beb23697, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/e2edfbe40b9e463f928d3f0b17ef252b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ccbdcfed166d4503b2a8e5f31dabcf45] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=47.7 K 2024-11-18T23:15:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T23:15:59,952 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting cc65c5377ee146c192f6c63335f45115, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731971753221 2024-11-18T23:15:59,956 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 395af88671ae4fcb9a4872b5beb23697, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731971754094 2024-11-18T23:15:59,960 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e2edfbe40b9e463f928d3f0b17ef252b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731971755378 2024-11-18T23:15:59,962 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:15:59,964 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ccbdcfed166d4503b2a8e5f31dabcf45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971757709 2024-11-18T23:15:59,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118de3d494522404599b7f235de0c5652cd_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971757844/Put/seqid=0 2024-11-18T23:15:59,997 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118d015a7369a334c56a80ae7f8cfb3e1b8_2312cbe93ba4cc288efe5635ed9f3f01 store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:00,000 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118d015a7369a334c56a80ae7f8cfb3e1b8_2312cbe93ba4cc288efe5635ed9f3f01, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:00,001 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118d015a7369a334c56a80ae7f8cfb3e1b8_2312cbe93ba4cc288efe5635ed9f3f01 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:00,006 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#B#compaction#133 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:00,007 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/1397a692a2e64f44abf079cc60513c25 is 50, key is test_row_0/B:col10/1731971757709/Put/seqid=0 2024-11-18T23:16:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:00,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:00,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741982_1158 (size=12304) 2024-11-18T23:16:00,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741984_1160 (size=12527) 2024-11-18T23:16:00,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741983_1159 (size=4469) 2024-11-18T23:16:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971820149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971820156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971820157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971820161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971820161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971820279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,286 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971820289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971820289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971820292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971820297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,300 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:00,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971820500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971820507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971820508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971820509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971820510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,520 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#A#compaction#131 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:00,521 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/9deeedf745974983bc9762ec5508b625 is 175, key is test_row_0/A:col10/1731971757709/Put/seqid=0 2024-11-18T23:16:00,542 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118de3d494522404599b7f235de0c5652cd_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118de3d494522404599b7f235de0c5652cd_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/1f5ec6ecc9064dcdaf0520d05ca29b5b, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:00,553 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/1397a692a2e64f44abf079cc60513c25 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/1397a692a2e64f44abf079cc60513c25 2024-11-18T23:16:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/1f5ec6ecc9064dcdaf0520d05ca29b5b is 175, key is test_row_0/A:col10/1731971757844/Put/seqid=0 2024-11-18T23:16:00,573 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/B of 2312cbe93ba4cc288efe5635ed9f3f01 into 1397a692a2e64f44abf079cc60513c25(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:00,573 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:00,573 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/B, priority=12, startTime=1731971759906; duration=0sec 2024-11-18T23:16:00,573 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:00,573 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:B 2024-11-18T23:16:00,573 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741985_1161 (size=31481) 2024-11-18T23:16:00,578 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:00,578 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/C is initiating minor compaction (all files) 2024-11-18T23:16:00,578 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/C in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:00,579 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/89351bbf098041b99cf50c7808e5eb64, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/777043d1e5384dabb8403620e49fef88, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/505fbc76313b462bb662f6d69336fb88, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/0af57539580141c583c0d672abad7d11] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=47.7 K 2024-11-18T23:16:00,580 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 89351bbf098041b99cf50c7808e5eb64, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731971753221 2024-11-18T23:16:00,582 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 777043d1e5384dabb8403620e49fef88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731971754094 2024-11-18T23:16:00,586 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 505fbc76313b462bb662f6d69336fb88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731971755378 2024-11-18T23:16:00,588 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 0af57539580141c583c0d672abad7d11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971757709 2024-11-18T23:16:00,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741986_1162 (size=31105) 2024-11-18T23:16:00,614 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/9deeedf745974983bc9762ec5508b625 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/9deeedf745974983bc9762ec5508b625 2024-11-18T23:16:00,616 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/1f5ec6ecc9064dcdaf0520d05ca29b5b 2024-11-18T23:16:00,626 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/A of 2312cbe93ba4cc288efe5635ed9f3f01 into 9deeedf745974983bc9762ec5508b625(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:00,626 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:00,626 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/A, priority=12, startTime=1731971759904; duration=0sec 2024-11-18T23:16:00,626 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:00,627 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:A 2024-11-18T23:16:00,651 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:00,652 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/deba27889a6d4f30b37ed5217de92f77 is 50, key is test_row_0/C:col10/1731971757709/Put/seqid=0 2024-11-18T23:16:00,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/9f3c712f02e64cb1b698bb94d5c448ea is 50, key is test_row_0/B:col10/1731971757844/Put/seqid=0 2024-11-18T23:16:00,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741987_1163 (size=12527) 2024-11-18T23:16:00,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741988_1164 (size=12151) 2024-11-18T23:16:00,806 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/9f3c712f02e64cb1b698bb94d5c448ea 2024-11-18T23:16:00,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971820807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,816 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/deba27889a6d4f30b37ed5217de92f77 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/deba27889a6d4f30b37ed5217de92f77 2024-11-18T23:16:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971820818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971820818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971820818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:00,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971820830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:00,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:00,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:00,861 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/C of 2312cbe93ba4cc288efe5635ed9f3f01 into deba27889a6d4f30b37ed5217de92f77(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:00,861 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:00,861 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/C, priority=12, startTime=1731971759906; duration=0sec 2024-11-18T23:16:00,861 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:00,861 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:C 2024-11-18T23:16:00,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/489e63492ec44854a77116b5ff496480 is 50, key is test_row_0/C:col10/1731971757844/Put/seqid=0 2024-11-18T23:16:00,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741989_1165 (size=12151) 2024-11-18T23:16:01,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971821328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,329 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:01,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:01,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971821328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:01,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:01,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971821328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:01,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971821328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:01,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:01,340 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/489e63492ec44854a77116b5ff496480 2024-11-18T23:16:01,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:01,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971821339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:01,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/1f5ec6ecc9064dcdaf0520d05ca29b5b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/1f5ec6ecc9064dcdaf0520d05ca29b5b 2024-11-18T23:16:01,402 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/1f5ec6ecc9064dcdaf0520d05ca29b5b, entries=150, sequenceid=235, filesize=30.4 K 2024-11-18T23:16:01,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/9f3c712f02e64cb1b698bb94d5c448ea as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9f3c712f02e64cb1b698bb94d5c448ea 2024-11-18T23:16:01,443 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9f3c712f02e64cb1b698bb94d5c448ea, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T23:16:01,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/489e63492ec44854a77116b5ff496480 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/489e63492ec44854a77116b5ff496480 2024-11-18T23:16:01,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/489e63492ec44854a77116b5ff496480, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T23:16:01,477 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1554ms, sequenceid=235, compaction requested=false 2024-11-18T23:16:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:01,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:01,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-18T23:16:01,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-18T23:16:01,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-18T23:16:01,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6910 sec 2024-11-18T23:16:01,527 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.7000 sec 2024-11-18T23:16:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-18T23:16:01,972 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:01,973 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-18T23:16:01,976 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T23:16:01,977 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:01,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T23:16:02,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-18T23:16:02,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:02,136 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T23:16:02,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:16:02,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:02,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:16:02,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:02,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:16:02,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:02,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188f967494c184495d8b832adb5be004ed_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971760149/Put/seqid=0 2024-11-18T23:16:02,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741990_1166 (size=12304) 2024-11-18T23:16:02,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:02,289 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188f967494c184495d8b832adb5be004ed_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188f967494c184495d8b832adb5be004ed_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:02,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/67f0df12751543ed824775abffd15b4f, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:02,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/67f0df12751543ed824775abffd15b4f is 175, key is test_row_0/A:col10/1731971760149/Put/seqid=0 2024-11-18T23:16:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T23:16:02,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741991_1167 (size=31105) 2024-11-18T23:16:02,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:02,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971822400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971822402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971822402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971822402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971822405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971822518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971822518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971822518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971822520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971822518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T23:16:02,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971822738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971822738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971822738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971822738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,750 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/67f0df12751543ed824775abffd15b4f 2024-11-18T23:16:02,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:02,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971822739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:02,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:02,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:02,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ed612bb97cb04c93896742ba918bb349 is 50, key is test_row_0/B:col10/1731971760149/Put/seqid=0 2024-11-18T23:16:02,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741992_1168 (size=12151) 2024-11-18T23:16:02,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ed612bb97cb04c93896742ba918bb349 2024-11-18T23:16:02,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/02190bd97e8b40989d649ba957a337e5 is 50, key is test_row_0/C:col10/1731971760149/Put/seqid=0 2024-11-18T23:16:02,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741993_1169 (size=12151) 2024-11-18T23:16:02,927 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/02190bd97e8b40989d649ba957a337e5 2024-11-18T23:16:02,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/67f0df12751543ed824775abffd15b4f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/67f0df12751543ed824775abffd15b4f 2024-11-18T23:16:02,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/67f0df12751543ed824775abffd15b4f, entries=150, sequenceid=252, filesize=30.4 K 2024-11-18T23:16:02,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ed612bb97cb04c93896742ba918bb349 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ed612bb97cb04c93896742ba918bb349 2024-11-18T23:16:02,973 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ed612bb97cb04c93896742ba918bb349, entries=150, sequenceid=252, filesize=11.9 K 2024-11-18T23:16:02,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/02190bd97e8b40989d649ba957a337e5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/02190bd97e8b40989d649ba957a337e5 2024-11-18T23:16:02,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/02190bd97e8b40989d649ba957a337e5, entries=150, sequenceid=252, filesize=11.9 K 2024-11-18T23:16:03,001 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2312cbe93ba4cc288efe5635ed9f3f01 in 865ms, sequenceid=252, compaction requested=true 2024-11-18T23:16:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-18T23:16:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-18T23:16:03,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-18T23:16:03,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0380 sec 2024-11-18T23:16:03,058 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T23:16:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:16:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:16:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:16:03,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:03,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:03,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.0790 sec 2024-11-18T23:16:03,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186475d9b15818492fb13f0a6ab889fa56_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971762399/Put/seqid=0 2024-11-18T23:16:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971823084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971823087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971823089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971823091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971823093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-18T23:16:03,118 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:03,119 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-18T23:16:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T23:16:03,126 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:03,127 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:03,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:03,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741994_1170 (size=12454) 2024-11-18T23:16:03,134 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:03,143 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186475d9b15818492fb13f0a6ab889fa56_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186475d9b15818492fb13f0a6ab889fa56_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:03,144 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/915fbc8796324f988742df180a9f66b2, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:03,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/915fbc8796324f988742df180a9f66b2 is 175, key is test_row_0/A:col10/1731971762399/Put/seqid=0 2024-11-18T23:16:03,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741995_1171 (size=31255) 2024-11-18T23:16:03,165 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/915fbc8796324f988742df180a9f66b2 2024-11-18T23:16:03,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/33c19508110d4a0eb6714ecbb99af893 is 50, key is test_row_0/B:col10/1731971762399/Put/seqid=0 2024-11-18T23:16:03,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741996_1172 (size=12301) 2024-11-18T23:16:03,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971823207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971823207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971823208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971823211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971823211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T23:16:03,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T23:16:03,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:03,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971823418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971823418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971823418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971823429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971823428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T23:16:03,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T23:16:03,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/33c19508110d4a0eb6714ecbb99af893 2024-11-18T23:16:03,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T23:16:03,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/da5250fe9253497b83289ec0fba748c4 is 50, key is test_row_0/C:col10/1731971762399/Put/seqid=0 2024-11-18T23:16:03,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741997_1173 (size=12301) 2024-11-18T23:16:03,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971823728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,735 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971823728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971823728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,736 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971823740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971823741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:03,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T23:16:03,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T23:16:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T23:16:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:03,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:03,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:04,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T23:16:04,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:04,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:04,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:04,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:04,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:04,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/da5250fe9253497b83289ec0fba748c4 2024-11-18T23:16:04,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/915fbc8796324f988742df180a9f66b2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/915fbc8796324f988742df180a9f66b2 2024-11-18T23:16:04,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/915fbc8796324f988742df180a9f66b2, entries=150, sequenceid=271, filesize=30.5 K 2024-11-18T23:16:04,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/33c19508110d4a0eb6714ecbb99af893 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/33c19508110d4a0eb6714ecbb99af893 2024-11-18T23:16:04,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/33c19508110d4a0eb6714ecbb99af893, entries=150, sequenceid=271, filesize=12.0 K 2024-11-18T23:16:04,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/da5250fe9253497b83289ec0fba748c4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/da5250fe9253497b83289ec0fba748c4 2024-11-18T23:16:04,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/da5250fe9253497b83289ec0fba748c4, entries=150, sequenceid=271, filesize=12.0 K 2024-11-18T23:16:04,176 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1119ms, sequenceid=271, compaction requested=true 2024-11-18T23:16:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:04,177 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:04,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:04,177 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:04,185 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:04,185 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/A is initiating minor compaction (all files) 2024-11-18T23:16:04,185 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/A in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:04,185 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/9deeedf745974983bc9762ec5508b625, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/1f5ec6ecc9064dcdaf0520d05ca29b5b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/67f0df12751543ed824775abffd15b4f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/915fbc8796324f988742df180a9f66b2] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=122.0 K 2024-11-18T23:16:04,185 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:04,185 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/9deeedf745974983bc9762ec5508b625, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/1f5ec6ecc9064dcdaf0520d05ca29b5b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/67f0df12751543ed824775abffd15b4f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/915fbc8796324f988742df180a9f66b2] 2024-11-18T23:16:04,188 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:04,189 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/B is initiating minor compaction (all files) 2024-11-18T23:16:04,189 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/B in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:04,189 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/1397a692a2e64f44abf079cc60513c25, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9f3c712f02e64cb1b698bb94d5c448ea, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ed612bb97cb04c93896742ba918bb349, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/33c19508110d4a0eb6714ecbb99af893] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=48.0 K 2024-11-18T23:16:04,189 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9deeedf745974983bc9762ec5508b625, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971757709 2024-11-18T23:16:04,189 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 1397a692a2e64f44abf079cc60513c25, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971757709 2024-11-18T23:16:04,190 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f5ec6ecc9064dcdaf0520d05ca29b5b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731971757825 2024-11-18T23:16:04,190 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f3c712f02e64cb1b698bb94d5c448ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731971757825 2024-11-18T23:16:04,192 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67f0df12751543ed824775abffd15b4f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731971760136 2024-11-18T23:16:04,192 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ed612bb97cb04c93896742ba918bb349, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731971760136 2024-11-18T23:16:04,193 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 915fbc8796324f988742df180a9f66b2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731971762399 2024-11-18T23:16:04,193 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 33c19508110d4a0eb6714ecbb99af893, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731971762399 2024-11-18T23:16:04,229 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:04,231 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#B#compaction#143 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:04,232 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/9fac13a0a48947238b9ca2bd1eb2e1fe is 50, key is test_row_0/B:col10/1731971762399/Put/seqid=0 2024-11-18T23:16:04,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-18T23:16:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:04,233 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:16:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:16:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:16:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:16:04,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:04,245 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111865197be2cfe84a3e8c8c5239a76b8fc6_2312cbe93ba4cc288efe5635ed9f3f01 store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:04,248 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111865197be2cfe84a3e8c8c5239a76b8fc6_2312cbe93ba4cc288efe5635ed9f3f01, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:04,248 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111865197be2cfe84a3e8c8c5239a76b8fc6_2312cbe93ba4cc288efe5635ed9f3f01 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:04,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:04,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T23:16:04,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111847ee34a5a27947a5b7852e61dbaa6299_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971763070/Put/seqid=0 2024-11-18T23:16:04,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741999_1175 (size=4469) 2024-11-18T23:16:04,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971824293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971824293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741998_1174 (size=12813) 2024-11-18T23:16:04,317 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#A#compaction#144 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:04,317 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/4dbc15cd5d864080b36592f6d7380fd7 is 175, key is test_row_0/A:col10/1731971762399/Put/seqid=0 2024-11-18T23:16:04,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971824309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971824311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971824311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742000_1176 (size=12454) 2024-11-18T23:16:04,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971824418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971824418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742001_1177 (size=31767) 2024-11-18T23:16:04,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971824427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971824427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971824427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,453 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/4dbc15cd5d864080b36592f6d7380fd7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4dbc15cd5d864080b36592f6d7380fd7 2024-11-18T23:16:04,479 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/A of 2312cbe93ba4cc288efe5635ed9f3f01 into 4dbc15cd5d864080b36592f6d7380fd7(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:04,479 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:04,479 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/A, priority=12, startTime=1731971764177; duration=0sec 2024-11-18T23:16:04,479 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:04,479 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:A 2024-11-18T23:16:04,479 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:04,488 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49130 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:04,488 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/C is initiating minor compaction (all files) 2024-11-18T23:16:04,488 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/C in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:04,489 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/deba27889a6d4f30b37ed5217de92f77, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/489e63492ec44854a77116b5ff496480, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/02190bd97e8b40989d649ba957a337e5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/da5250fe9253497b83289ec0fba748c4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=48.0 K 2024-11-18T23:16:04,489 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting deba27889a6d4f30b37ed5217de92f77, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1731971757709 2024-11-18T23:16:04,489 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 489e63492ec44854a77116b5ff496480, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731971757825 2024-11-18T23:16:04,490 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02190bd97e8b40989d649ba957a337e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731971760136 2024-11-18T23:16:04,490 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5250fe9253497b83289ec0fba748c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731971762399 2024-11-18T23:16:04,511 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:04,512 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/f9123e0a842d45af894efa0bbbedf93d is 50, key is test_row_0/C:col10/1731971762399/Put/seqid=0 2024-11-18T23:16:04,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742002_1178 (size=12813) 2024-11-18T23:16:04,584 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/f9123e0a842d45af894efa0bbbedf93d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/f9123e0a842d45af894efa0bbbedf93d 2024-11-18T23:16:04,594 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/C of 2312cbe93ba4cc288efe5635ed9f3f01 into f9123e0a842d45af894efa0bbbedf93d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:04,594 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:04,594 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/C, priority=12, startTime=1731971764177; duration=0sec 2024-11-18T23:16:04,594 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:04,594 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:C 2024-11-18T23:16:04,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971824627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971824627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971824637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971824637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971824647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,749 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/9fac13a0a48947238b9ca2bd1eb2e1fe as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9fac13a0a48947238b9ca2bd1eb2e1fe 2024-11-18T23:16:04,757 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/B of 2312cbe93ba4cc288efe5635ed9f3f01 into 9fac13a0a48947238b9ca2bd1eb2e1fe(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:04,757 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:04,757 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/B, priority=12, startTime=1731971764177; duration=0sec 2024-11-18T23:16:04,757 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:04,757 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:B 2024-11-18T23:16:04,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:04,783 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111847ee34a5a27947a5b7852e61dbaa6299_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111847ee34a5a27947a5b7852e61dbaa6299_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:04,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a is 175, key is test_row_0/A:col10/1731971763070/Put/seqid=0 2024-11-18T23:16:04,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742003_1179 (size=31255) 2024-11-18T23:16:04,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971824953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,960 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971824953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971824959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971824959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:04,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:04,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971824959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:04,972 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:05,253 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a 2024-11-18T23:16:05,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/135f3b6f8a544313917a13ccaa224eaf is 50, key is test_row_0/B:col10/1731971763070/Put/seqid=0 2024-11-18T23:16:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T23:16:05,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742004_1180 (size=12301) 2024-11-18T23:16:05,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971825467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:05,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971825467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:05,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971825468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:05,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971825478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:05,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971825479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:05,691 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/135f3b6f8a544313917a13ccaa224eaf 2024-11-18T23:16:05,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/c5b37019e107495cba85ffde42529c7a is 50, key is test_row_0/C:col10/1731971763070/Put/seqid=0 2024-11-18T23:16:05,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742005_1181 (size=12301) 2024-11-18T23:16:05,723 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/c5b37019e107495cba85ffde42529c7a 2024-11-18T23:16:05,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a 2024-11-18T23:16:05,738 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a, entries=150, sequenceid=284, filesize=30.5 K 2024-11-18T23:16:05,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/135f3b6f8a544313917a13ccaa224eaf as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/135f3b6f8a544313917a13ccaa224eaf 2024-11-18T23:16:05,754 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/135f3b6f8a544313917a13ccaa224eaf, entries=150, sequenceid=284, filesize=12.0 K 2024-11-18T23:16:05,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/c5b37019e107495cba85ffde42529c7a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/c5b37019e107495cba85ffde42529c7a 2024-11-18T23:16:05,763 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/c5b37019e107495cba85ffde42529c7a, entries=150, sequenceid=284, filesize=12.0 K 2024-11-18T23:16:05,764 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1531ms, sequenceid=284, compaction requested=false 2024-11-18T23:16:05,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:05,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:05,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-18T23:16:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-18T23:16:05,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-18T23:16:05,771 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6430 sec 2024-11-18T23:16:05,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.6530 sec 2024-11-18T23:16:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:06,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-18T23:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:06,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188fabff8c2ea148c4837ff926dfd0e588_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971766486/Put/seqid=0 2024-11-18T23:16:06,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971826612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971826614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971826632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971826636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971826641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742006_1182 (size=14994) 2024-11-18T23:16:06,698 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:06,710 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188fabff8c2ea148c4837ff926dfd0e588_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fabff8c2ea148c4837ff926dfd0e588_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:06,718 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f94ff6c79caf455e9588b6f7670d78ed, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:06,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f94ff6c79caf455e9588b6f7670d78ed is 175, key is test_row_0/A:col10/1731971766486/Put/seqid=0 2024-11-18T23:16:06,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971826747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971826747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742007_1183 (size=39949) 2024-11-18T23:16:06,769 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f94ff6c79caf455e9588b6f7670d78ed 2024-11-18T23:16:06,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971826764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,776 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971826764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971826768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/f6ea6d3e775648bc9eaf4a3a5187dda4 is 50, key is test_row_0/B:col10/1731971766486/Put/seqid=0 2024-11-18T23:16:06,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742008_1184 (size=12301) 2024-11-18T23:16:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971826968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971826968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971826988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971826988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971826988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:06,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:07,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/f6ea6d3e775648bc9eaf4a3a5187dda4 2024-11-18T23:16:07,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/fee91ce464884b0c80c97c2090e552c2 is 50, key is test_row_0/C:col10/1731971766486/Put/seqid=0 2024-11-18T23:16:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971827279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:07,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:07,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971827292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-18T23:16:07,307 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:07,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-18T23:16:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T23:16:07,316 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:07,317 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:07,317 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:07,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:07,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971827309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:07,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971827312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971827309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:07,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742009_1185 (size=12301) 2024-11-18T23:16:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T23:16:07,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T23:16:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:07,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T23:16:07,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T23:16:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:07,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:07,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/fee91ce464884b0c80c97c2090e552c2 2024-11-18T23:16:07,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/f94ff6c79caf455e9588b6f7670d78ed as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f94ff6c79caf455e9588b6f7670d78ed 2024-11-18T23:16:07,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f94ff6c79caf455e9588b6f7670d78ed, entries=200, sequenceid=309, filesize=39.0 K 2024-11-18T23:16:07,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/f6ea6d3e775648bc9eaf4a3a5187dda4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f6ea6d3e775648bc9eaf4a3a5187dda4 2024-11-18T23:16:07,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f6ea6d3e775648bc9eaf4a3a5187dda4, entries=150, sequenceid=309, filesize=12.0 K 2024-11-18T23:16:07,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/fee91ce464884b0c80c97c2090e552c2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/fee91ce464884b0c80c97c2090e552c2 2024-11-18T23:16:07,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/fee91ce464884b0c80c97c2090e552c2, entries=150, sequenceid=309, filesize=12.0 K 2024-11-18T23:16:07,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1307ms, sequenceid=309, compaction requested=true 2024-11-18T23:16:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:07,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T23:16:07,802 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:07,808 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:07,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-18T23:16:07,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,809 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:16:07,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:16:07,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:07,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:16:07,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:07,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:16:07,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:07,813 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:07,813 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/A is initiating minor compaction (all files) 2024-11-18T23:16:07,813 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/A in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,813 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4dbc15cd5d864080b36592f6d7380fd7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f94ff6c79caf455e9588b6f7670d78ed] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=100.6 K 2024-11-18T23:16:07,813 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,813 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4dbc15cd5d864080b36592f6d7380fd7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f94ff6c79caf455e9588b6f7670d78ed] 2024-11-18T23:16:07,814 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:07,814 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/C is initiating minor compaction (all files) 2024-11-18T23:16:07,814 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/C in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:07,814 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/f9123e0a842d45af894efa0bbbedf93d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/c5b37019e107495cba85ffde42529c7a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/fee91ce464884b0c80c97c2090e552c2] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=36.5 K 2024-11-18T23:16:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:07,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. as already flushing 2024-11-18T23:16:07,821 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dbc15cd5d864080b36592f6d7380fd7, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731971762399 2024-11-18T23:16:07,821 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f9123e0a842d45af894efa0bbbedf93d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731971762399 2024-11-18T23:16:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,829 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b01f02e0d5ab4a9b91a1d76ecf6f5a7a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731971763070 2024-11-18T23:16:07,830 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c5b37019e107495cba85ffde42529c7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731971763070 2024-11-18T23:16:07,830 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f94ff6c79caf455e9588b6f7670d78ed, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731971764301 2024-11-18T23:16:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,836 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting fee91ce464884b0c80c97c2090e552c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731971764302 2024-11-18T23:16:07,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118103c7043ecc946be876abdd67ef1e12c_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971766616/Put/seqid=0 2024-11-18T23:16:07,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,860 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:07,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,873 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111821d694cba3d44c188452ecbb80cfc78f_2312cbe93ba4cc288efe5635ed9f3f01 store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:07,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,875 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111821d694cba3d44c188452ecbb80cfc78f_2312cbe93ba4cc288efe5635ed9f3f01, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:07,875 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111821d694cba3d44c188452ecbb80cfc78f_2312cbe93ba4cc288efe5635ed9f3f01 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,905 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#C#compaction#154 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:07,906 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/5ad9772d7e3b48588d2a0418921150ba is 50, key is test_row_0/C:col10/1731971766486/Put/seqid=0 2024-11-18T23:16:07,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742010_1186 (size=9914) 2024-11-18T23:16:07,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T23:16:07,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,952 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118103c7043ecc946be876abdd67ef1e12c_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118103c7043ecc946be876abdd67ef1e12c_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:07,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/c7e3f9371f244fde826e595b3d11ed69, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:07,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/c7e3f9371f244fde826e595b3d11ed69 is 175, key is test_row_0/A:col10/1731971766616/Put/seqid=0 2024-11-18T23:16:07,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731971767810, max timestamp=1731971767969Num uniques 859; 2024-11-18T23:16:07,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T23:16:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731971767810, max timestamp=1731971767969Num uniques 859; 2024-11-18T23:16:07,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T23:16:07,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731971767810, max timestamp=1731971767969Num uniques 859; 2024-11-18T23:16:07,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-18T23:16:07,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:07,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971827976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742012_1188 (size=12915) 2024-11-18T23:16:08,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971827984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742013_1189 (size=22561) 2024-11-18T23:16:08,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971827985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971827996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971828002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742011_1187 (size=4469) 2024-11-18T23:16:08,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971828108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971828127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971828133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971828127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971828138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,141 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971828317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971828348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971828349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971828350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971828352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,414 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/c7e3f9371f244fde826e595b3d11ed69 2024-11-18T23:16:08,419 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/5ad9772d7e3b48588d2a0418921150ba as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/5ad9772d7e3b48588d2a0418921150ba 2024-11-18T23:16:08,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/7ff58bdb7b0e4b108e8866d1307426e7 is 50, key is test_row_0/B:col10/1731971766616/Put/seqid=0 2024-11-18T23:16:08,433 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/C of 2312cbe93ba4cc288efe5635ed9f3f01 into 5ad9772d7e3b48588d2a0418921150ba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:08,433 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:08,433 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/C, priority=13, startTime=1731971767801; duration=0sec 2024-11-18T23:16:08,433 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:08,433 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:C 2024-11-18T23:16:08,433 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:08,443 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#A#compaction#153 average throughput is 0.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:08,443 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:08,443 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 2312cbe93ba4cc288efe5635ed9f3f01/B is initiating minor compaction (all files) 2024-11-18T23:16:08,444 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2312cbe93ba4cc288efe5635ed9f3f01/B in TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:08,444 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9fac13a0a48947238b9ca2bd1eb2e1fe, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/135f3b6f8a544313917a13ccaa224eaf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f6ea6d3e775648bc9eaf4a3a5187dda4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp, totalSize=36.5 K 2024-11-18T23:16:08,444 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/e779819e7ad441e29d4593dd68ab8567 is 175, key is test_row_0/A:col10/1731971766486/Put/seqid=0 2024-11-18T23:16:08,444 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fac13a0a48947238b9ca2bd1eb2e1fe, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731971762399 2024-11-18T23:16:08,450 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 135f3b6f8a544313917a13ccaa224eaf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731971763070 2024-11-18T23:16:08,451 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ea6d3e775648bc9eaf4a3a5187dda4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731971764302 2024-11-18T23:16:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T23:16:08,495 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2312cbe93ba4cc288efe5635ed9f3f01#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:08,496 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/bd8f7ba89bd0463ab8fd05ec2370a947 is 50, key is test_row_0/B:col10/1731971766486/Put/seqid=0 2024-11-18T23:16:08,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742015_1191 (size=31869) 2024-11-18T23:16:08,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742014_1190 (size=9857) 2024-11-18T23:16:08,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/7ff58bdb7b0e4b108e8866d1307426e7 2024-11-18T23:16:08,539 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/e779819e7ad441e29d4593dd68ab8567 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e779819e7ad441e29d4593dd68ab8567 2024-11-18T23:16:08,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742016_1192 (size=12915) 2024-11-18T23:16:08,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/4880512f0a3949adb17d1d98540a405e is 50, key is test_row_0/C:col10/1731971766616/Put/seqid=0 2024-11-18T23:16:08,585 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/A of 2312cbe93ba4cc288efe5635ed9f3f01 into e779819e7ad441e29d4593dd68ab8567(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:08,585 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:08,585 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/A, priority=13, startTime=1731971767801; duration=0sec 2024-11-18T23:16:08,585 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:08,585 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:A 2024-11-18T23:16:08,594 INFO [Thread-597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-597. 2024-11-18T23:16:08,594 DEBUG [Thread-597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:08,594 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,594 DEBUG [Thread-597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:08,602 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/bd8f7ba89bd0463ab8fd05ec2370a947 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/bd8f7ba89bd0463ab8fd05ec2370a947 2024-11-18T23:16:08,607 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-18T23:16:08,607 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:08,607 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,607 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:08,615 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-18T23:16:08,616 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:08,616 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,616 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:08,625 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-18T23:16:08,625 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:08,625 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,625 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:08,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:08,628 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2312cbe93ba4cc288efe5635ed9f3f01/B of 2312cbe93ba4cc288efe5635ed9f3f01 into bd8f7ba89bd0463ab8fd05ec2370a947(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:08,628 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:08,628 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., storeName=2312cbe93ba4cc288efe5635ed9f3f01/B, priority=13, startTime=1731971767801; duration=0sec 2024-11-18T23:16:08,629 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:08,629 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:B 2024-11-18T23:16:08,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742017_1193 (size=9857) 2024-11-18T23:16:08,640 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/4880512f0a3949adb17d1d98540a405e 2024-11-18T23:16:08,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48858 deadline: 1731971828641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/c7e3f9371f244fde826e595b3d11ed69 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/c7e3f9371f244fde826e595b3d11ed69 2024-11-18T23:16:08,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48848 deadline: 1731971828660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48898 deadline: 1731971828660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48914 deadline: 1731971828671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48836 deadline: 1731971828672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:08,673 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/c7e3f9371f244fde826e595b3d11ed69, entries=100, sequenceid=320, filesize=22.0 K 2024-11-18T23:16:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:08,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/7ff58bdb7b0e4b108e8866d1307426e7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/7ff58bdb7b0e4b108e8866d1307426e7 2024-11-18T23:16:08,695 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/7ff58bdb7b0e4b108e8866d1307426e7, entries=100, sequenceid=320, filesize=9.6 K 2024-11-18T23:16:08,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/4880512f0a3949adb17d1d98540a405e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/4880512f0a3949adb17d1d98540a405e 2024-11-18T23:16:08,702 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/4880512f0a3949adb17d1d98540a405e, entries=100, sequenceid=320, filesize=9.6 K 2024-11-18T23:16:08,708 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=147.60 KB/151140 for 2312cbe93ba4cc288efe5635ed9f3f01 in 899ms, sequenceid=320, compaction requested=false 2024-11-18T23:16:08,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:08,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:08,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-18T23:16:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-18T23:16:08,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-18T23:16:08,742 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4070 sec 2024-11-18T23:16:08,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4350 sec 2024-11-18T23:16:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:09,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-18T23:16:09,149 INFO [Thread-590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-590. 2024-11-18T23:16:09,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:16:09,149 DEBUG [Thread-590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:09,149 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:16:09,150 DEBUG [Thread-590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:16:09,150 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:09,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:09,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180fdb4fbbf2434d999293b32e06a49a40_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_0/A:col10/1731971767986/Put/seqid=0 2024-11-18T23:16:09,174 INFO [Thread-592 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-592. 2024-11-18T23:16:09,174 DEBUG [Thread-592 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:09,174 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,174 DEBUG [Thread-592 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:09,189 INFO [Thread-594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-594. 2024-11-18T23:16:09,189 DEBUG [Thread-594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:09,189 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,189 DEBUG [Thread-594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,189 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:09,190 INFO [Thread-586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-586. 2024-11-18T23:16:09,190 DEBUG [Thread-586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:09,191 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,191 DEBUG [Thread-586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:09,192 INFO [Thread-588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-588. 2024-11-18T23:16:09,192 DEBUG [Thread-588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:09,192 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,192 DEBUG [Thread-588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,193 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:09,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742018_1194 (size=12454) 2024-11-18T23:16:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-18T23:16:09,488 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:09,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T23:16:09,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-18T23:16:09,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-18T23:16:09,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-18T23:16:09,488 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1389 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1343 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 648 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1941 rows 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 645 2024-11-18T23:16:09,489 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1934 rows 2024-11-18T23:16:09,489 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:16:09,489 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:16:09,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:09,489 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:09,490 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T23:16:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T23:16:09,494 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971769494"}]},"ts":"1731971769494"} 2024-11-18T23:16:09,496 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T23:16:09,496 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T23:16:09,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:16:09,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, UNASSIGN}] 2024-11-18T23:16:09,499 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, UNASSIGN 2024-11-18T23:16:09,500 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:09,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, UNASSIGN because future has completed 2024-11-18T23:16:09,502 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:16:09,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:16:09,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:09,600 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180fdb4fbbf2434d999293b32e06a49a40_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180fdb4fbbf2434d999293b32e06a49a40_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:09,601 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/2ec2591bd03f49fbbed42ac87cf042d8, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:09,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/2ec2591bd03f49fbbed42ac87cf042d8 is 175, key is test_row_0/A:col10/1731971767986/Put/seqid=0 2024-11-18T23:16:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T23:16:09,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742019_1195 (size=31255) 2024-11-18T23:16:09,619 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/2ec2591bd03f49fbbed42ac87cf042d8 2024-11-18T23:16:09,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ad07ebc4bc694515823d70b501b9edfb is 50, key is test_row_0/B:col10/1731971767986/Put/seqid=0 2024-11-18T23:16:09,655 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:09,655 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:16:09,655 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 2312cbe93ba4cc288efe5635ed9f3f01, disabling compactions & flushes 2024-11-18T23:16:09,656 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:09,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742020_1196 (size=12301) 2024-11-18T23:16:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T23:16:10,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ad07ebc4bc694515823d70b501b9edfb 2024-11-18T23:16:10,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/a0eaa18d93de48c38881607799898cff is 50, key is test_row_0/C:col10/1731971767986/Put/seqid=0 2024-11-18T23:16:10,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742021_1197 (size=12301) 2024-11-18T23:16:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T23:16:10,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/a0eaa18d93de48c38881607799898cff 2024-11-18T23:16:10,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/2ec2591bd03f49fbbed42ac87cf042d8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2ec2591bd03f49fbbed42ac87cf042d8 2024-11-18T23:16:10,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2ec2591bd03f49fbbed42ac87cf042d8, entries=150, sequenceid=349, filesize=30.5 K 2024-11-18T23:16:10,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/ad07ebc4bc694515823d70b501b9edfb as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ad07ebc4bc694515823d70b501b9edfb 2024-11-18T23:16:10,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ad07ebc4bc694515823d70b501b9edfb, entries=150, sequenceid=349, filesize=12.0 K 2024-11-18T23:16:10,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/a0eaa18d93de48c38881607799898cff as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a0eaa18d93de48c38881607799898cff 2024-11-18T23:16:10,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a0eaa18d93de48c38881607799898cff, entries=150, sequenceid=349, filesize=12.0 K 2024-11-18T23:16:10,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=26.84 KB/27480 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1355ms, sequenceid=349, compaction requested=true 2024-11-18T23:16:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2312cbe93ba4cc288efe5635ed9f3f01: 2024-11-18T23:16:10,504 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:10,504 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:10,504 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. because compaction request was cancelled 2024-11-18T23:16:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:10,504 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:A 2024-11-18T23:16:10,504 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. after waiting 0 ms 2024-11-18T23:16:10,504 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:10,504 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. because compaction request was cancelled 2024-11-18T23:16:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:B, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:10,504 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:B 2024-11-18T23:16:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2312cbe93ba4cc288efe5635ed9f3f01:C, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:10,504 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. because compaction request was cancelled 2024-11-18T23:16:10,504 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 2312cbe93ba4cc288efe5635ed9f3f01 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T23:16:10,504 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2312cbe93ba4cc288efe5635ed9f3f01:C 2024-11-18T23:16:10,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:10,504 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=A 2024-11-18T23:16:10,505 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:10,505 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=B 2024-11-18T23:16:10,505 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:10,505 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2312cbe93ba4cc288efe5635ed9f3f01, store=C 2024-11-18T23:16:10,505 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:10,511 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c6fbba4b52fb41daa5deba599b5858be_2312cbe93ba4cc288efe5635ed9f3f01 is 50, key is test_row_1/A:col10/1731971769188/Put/seqid=0 2024-11-18T23:16:10,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742022_1198 (size=9914) 2024-11-18T23:16:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T23:16:10,919 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:10,925 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c6fbba4b52fb41daa5deba599b5858be_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c6fbba4b52fb41daa5deba599b5858be_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:10,926 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/4e3b995fcf6b483194edb5c84e2a876d, store: [table=TestAcidGuarantees family=A region=2312cbe93ba4cc288efe5635ed9f3f01] 2024-11-18T23:16:10,927 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/4e3b995fcf6b483194edb5c84e2a876d is 175, key is test_row_1/A:col10/1731971769188/Put/seqid=0 2024-11-18T23:16:10,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742023_1199 (size=22561) 2024-11-18T23:16:11,335 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/4e3b995fcf6b483194edb5c84e2a876d 2024-11-18T23:16:11,346 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/c7e9da3f96c141759caaa1c8e0cd0a87 is 50, key is test_row_1/B:col10/1731971769188/Put/seqid=0 2024-11-18T23:16:11,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742024_1200 (size=9857) 2024-11-18T23:16:11,354 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/c7e9da3f96c141759caaa1c8e0cd0a87 2024-11-18T23:16:11,376 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/d52dafa310074d01bff1e6c45b09de0d is 50, key is test_row_1/C:col10/1731971769188/Put/seqid=0 2024-11-18T23:16:11,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742025_1201 (size=9857) 2024-11-18T23:16:11,466 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T23:16:11,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T23:16:11,786 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/d52dafa310074d01bff1e6c45b09de0d 2024-11-18T23:16:11,796 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/A/4e3b995fcf6b483194edb5c84e2a876d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4e3b995fcf6b483194edb5c84e2a876d 2024-11-18T23:16:11,810 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4e3b995fcf6b483194edb5c84e2a876d, entries=100, sequenceid=356, filesize=22.0 K 2024-11-18T23:16:11,812 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/B/c7e9da3f96c141759caaa1c8e0cd0a87 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c7e9da3f96c141759caaa1c8e0cd0a87 2024-11-18T23:16:11,832 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c7e9da3f96c141759caaa1c8e0cd0a87, entries=100, sequenceid=356, filesize=9.6 K 2024-11-18T23:16:11,841 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/.tmp/C/d52dafa310074d01bff1e6c45b09de0d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d52dafa310074d01bff1e6c45b09de0d 2024-11-18T23:16:11,858 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d52dafa310074d01bff1e6c45b09de0d, entries=100, sequenceid=356, filesize=9.6 K 2024-11-18T23:16:11,862 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1357ms, sequenceid=356, compaction requested=true 2024-11-18T23:16:11,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d3601f943fee440dac7a7e51c0ecc8bc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6ff405017aed4e3db2e4ad0e0c617366, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6a609a0e3b1e417bb4aa0c7df0e41af9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2933ba0e2b2c47fea4962f71f3622686, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d0bed6f7af0a46d2bbb643c2c6631589, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e618345965e64814ae5ec3b4033fdb3d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/624a5fc0023c490d9f1e9ee04180f1d8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/07d93b044a924a6aad079ee30fb65601, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b20f0a80f7ea4a5ba505a98bcb494fe3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f40cfd1e9ded4843a30946757a281b8a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/9deeedf745974983bc9762ec5508b625, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/765bce4c322c45e49a2ed2add3c9984c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/1f5ec6ecc9064dcdaf0520d05ca29b5b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/67f0df12751543ed824775abffd15b4f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4dbc15cd5d864080b36592f6d7380fd7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/915fbc8796324f988742df180a9f66b2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f94ff6c79caf455e9588b6f7670d78ed] to archive 2024-11-18T23:16:11,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:16:11,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d3601f943fee440dac7a7e51c0ecc8bc to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d3601f943fee440dac7a7e51c0ecc8bc 2024-11-18T23:16:11,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6ff405017aed4e3db2e4ad0e0c617366 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6ff405017aed4e3db2e4ad0e0c617366 2024-11-18T23:16:11,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6a609a0e3b1e417bb4aa0c7df0e41af9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/6a609a0e3b1e417bb4aa0c7df0e41af9 2024-11-18T23:16:11,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2933ba0e2b2c47fea4962f71f3622686 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2933ba0e2b2c47fea4962f71f3622686 2024-11-18T23:16:11,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d0bed6f7af0a46d2bbb643c2c6631589 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/d0bed6f7af0a46d2bbb643c2c6631589 2024-11-18T23:16:11,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e618345965e64814ae5ec3b4033fdb3d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e618345965e64814ae5ec3b4033fdb3d 2024-11-18T23:16:11,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/624a5fc0023c490d9f1e9ee04180f1d8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/624a5fc0023c490d9f1e9ee04180f1d8 2024-11-18T23:16:11,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/07d93b044a924a6aad079ee30fb65601 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/07d93b044a924a6aad079ee30fb65601 2024-11-18T23:16:11,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b20f0a80f7ea4a5ba505a98bcb494fe3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b20f0a80f7ea4a5ba505a98bcb494fe3 2024-11-18T23:16:11,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f40cfd1e9ded4843a30946757a281b8a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f40cfd1e9ded4843a30946757a281b8a 2024-11-18T23:16:11,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/9deeedf745974983bc9762ec5508b625 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/9deeedf745974983bc9762ec5508b625 2024-11-18T23:16:11,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/765bce4c322c45e49a2ed2add3c9984c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/765bce4c322c45e49a2ed2add3c9984c 2024-11-18T23:16:11,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/1f5ec6ecc9064dcdaf0520d05ca29b5b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/1f5ec6ecc9064dcdaf0520d05ca29b5b 2024-11-18T23:16:11,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/67f0df12751543ed824775abffd15b4f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/67f0df12751543ed824775abffd15b4f 2024-11-18T23:16:11,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4dbc15cd5d864080b36592f6d7380fd7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4dbc15cd5d864080b36592f6d7380fd7 2024-11-18T23:16:11,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/915fbc8796324f988742df180a9f66b2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/915fbc8796324f988742df180a9f66b2 2024-11-18T23:16:11,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/b01f02e0d5ab4a9b91a1d76ecf6f5a7a 2024-11-18T23:16:11,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f94ff6c79caf455e9588b6f7670d78ed to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/f94ff6c79caf455e9588b6f7670d78ed 2024-11-18T23:16:11,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/326dcbbbe28947b2bc7d7e939b07cd77, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f0795523c19e4f24be76977a2c9ca667, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/4ae967cdcd57442fa4482c2438115311, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c25d7a40b3aa4e0ebb989608158fd99c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/50968f5306bc45f98363903d35f0bbe5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/13755b8023b744e1ab52e7f60901788f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cc65c5377ee146c192f6c63335f45115, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cf8a6dcd8102434cb2bf76e386538fd4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/395af88671ae4fcb9a4872b5beb23697, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/e2edfbe40b9e463f928d3f0b17ef252b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/1397a692a2e64f44abf079cc60513c25, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ccbdcfed166d4503b2a8e5f31dabcf45, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9f3c712f02e64cb1b698bb94d5c448ea, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ed612bb97cb04c93896742ba918bb349, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9fac13a0a48947238b9ca2bd1eb2e1fe, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/33c19508110d4a0eb6714ecbb99af893, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/135f3b6f8a544313917a13ccaa224eaf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f6ea6d3e775648bc9eaf4a3a5187dda4] to archive 2024-11-18T23:16:11,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:16:11,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/326dcbbbe28947b2bc7d7e939b07cd77 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/326dcbbbe28947b2bc7d7e939b07cd77 2024-11-18T23:16:11,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f0795523c19e4f24be76977a2c9ca667 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f0795523c19e4f24be76977a2c9ca667 2024-11-18T23:16:11,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/4ae967cdcd57442fa4482c2438115311 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/4ae967cdcd57442fa4482c2438115311 2024-11-18T23:16:11,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c25d7a40b3aa4e0ebb989608158fd99c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c25d7a40b3aa4e0ebb989608158fd99c 2024-11-18T23:16:11,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/50968f5306bc45f98363903d35f0bbe5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/50968f5306bc45f98363903d35f0bbe5 2024-11-18T23:16:11,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/13755b8023b744e1ab52e7f60901788f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/13755b8023b744e1ab52e7f60901788f 2024-11-18T23:16:11,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cc65c5377ee146c192f6c63335f45115 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cc65c5377ee146c192f6c63335f45115 2024-11-18T23:16:11,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cf8a6dcd8102434cb2bf76e386538fd4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/cf8a6dcd8102434cb2bf76e386538fd4 2024-11-18T23:16:11,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/395af88671ae4fcb9a4872b5beb23697 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/395af88671ae4fcb9a4872b5beb23697 2024-11-18T23:16:11,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/e2edfbe40b9e463f928d3f0b17ef252b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/e2edfbe40b9e463f928d3f0b17ef252b 2024-11-18T23:16:11,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/1397a692a2e64f44abf079cc60513c25 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/1397a692a2e64f44abf079cc60513c25 2024-11-18T23:16:11,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ccbdcfed166d4503b2a8e5f31dabcf45 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ccbdcfed166d4503b2a8e5f31dabcf45 2024-11-18T23:16:11,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9f3c712f02e64cb1b698bb94d5c448ea to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9f3c712f02e64cb1b698bb94d5c448ea 2024-11-18T23:16:11,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ed612bb97cb04c93896742ba918bb349 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ed612bb97cb04c93896742ba918bb349 2024-11-18T23:16:11,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9fac13a0a48947238b9ca2bd1eb2e1fe to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/9fac13a0a48947238b9ca2bd1eb2e1fe 2024-11-18T23:16:11,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/33c19508110d4a0eb6714ecbb99af893 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/33c19508110d4a0eb6714ecbb99af893 2024-11-18T23:16:11,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/135f3b6f8a544313917a13ccaa224eaf to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/135f3b6f8a544313917a13ccaa224eaf 2024-11-18T23:16:11,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f6ea6d3e775648bc9eaf4a3a5187dda4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/f6ea6d3e775648bc9eaf4a3a5187dda4 2024-11-18T23:16:11,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/cfc58943861b46caa35f0c2108e15c80, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d79ba7f95ee3478e87e9d00be5309b2a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a015805562ce4e2aa775117536e72864, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/701f86e608ef47518cc75e0f399d5f45, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/29c84aa0ec674a9c8bf16fff31e2c49d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/2ac8b24bd08c46e28346907ca6da4aa1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/89351bbf098041b99cf50c7808e5eb64, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/7fe232948289475dbd4861f1f719a95e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/777043d1e5384dabb8403620e49fef88, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/505fbc76313b462bb662f6d69336fb88, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/deba27889a6d4f30b37ed5217de92f77, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/0af57539580141c583c0d672abad7d11, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/489e63492ec44854a77116b5ff496480, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/02190bd97e8b40989d649ba957a337e5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/f9123e0a842d45af894efa0bbbedf93d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/da5250fe9253497b83289ec0fba748c4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/c5b37019e107495cba85ffde42529c7a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/fee91ce464884b0c80c97c2090e552c2] to archive 2024-11-18T23:16:12,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:16:12,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/cfc58943861b46caa35f0c2108e15c80 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/cfc58943861b46caa35f0c2108e15c80 2024-11-18T23:16:12,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d79ba7f95ee3478e87e9d00be5309b2a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d79ba7f95ee3478e87e9d00be5309b2a 2024-11-18T23:16:12,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a015805562ce4e2aa775117536e72864 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a015805562ce4e2aa775117536e72864 2024-11-18T23:16:12,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/701f86e608ef47518cc75e0f399d5f45 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/701f86e608ef47518cc75e0f399d5f45 2024-11-18T23:16:12,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/29c84aa0ec674a9c8bf16fff31e2c49d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/29c84aa0ec674a9c8bf16fff31e2c49d 2024-11-18T23:16:12,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/2ac8b24bd08c46e28346907ca6da4aa1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/2ac8b24bd08c46e28346907ca6da4aa1 2024-11-18T23:16:12,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/89351bbf098041b99cf50c7808e5eb64 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/89351bbf098041b99cf50c7808e5eb64 2024-11-18T23:16:12,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/7fe232948289475dbd4861f1f719a95e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/7fe232948289475dbd4861f1f719a95e 2024-11-18T23:16:12,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/777043d1e5384dabb8403620e49fef88 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/777043d1e5384dabb8403620e49fef88 2024-11-18T23:16:12,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/505fbc76313b462bb662f6d69336fb88 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/505fbc76313b462bb662f6d69336fb88 2024-11-18T23:16:12,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/deba27889a6d4f30b37ed5217de92f77 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/deba27889a6d4f30b37ed5217de92f77 2024-11-18T23:16:12,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/0af57539580141c583c0d672abad7d11 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/0af57539580141c583c0d672abad7d11 2024-11-18T23:16:12,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/489e63492ec44854a77116b5ff496480 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/489e63492ec44854a77116b5ff496480 2024-11-18T23:16:12,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/02190bd97e8b40989d649ba957a337e5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/02190bd97e8b40989d649ba957a337e5 2024-11-18T23:16:12,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/f9123e0a842d45af894efa0bbbedf93d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/f9123e0a842d45af894efa0bbbedf93d 2024-11-18T23:16:12,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/da5250fe9253497b83289ec0fba748c4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/da5250fe9253497b83289ec0fba748c4 2024-11-18T23:16:12,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/c5b37019e107495cba85ffde42529c7a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/c5b37019e107495cba85ffde42529c7a 2024-11-18T23:16:12,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/fee91ce464884b0c80c97c2090e552c2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/fee91ce464884b0c80c97c2090e552c2 2024-11-18T23:16:12,096 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/recovered.edits/359.seqid, newMaxSeqId=359, maxSeqId=4 2024-11-18T23:16:12,102 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. 2024-11-18T23:16:12,103 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 2312cbe93ba4cc288efe5635ed9f3f01: Waiting for close lock at 1731971769655Running coprocessor pre-close hooks at 1731971769655Disabling compacts and flushes for region at 1731971769655Disabling writes for close at 1731971770504 (+849 ms)Obtaining lock to block concurrent updates at 1731971770504Preparing flush snapshotting stores in 2312cbe93ba4cc288efe5635ed9f3f01 at 1731971770504Finished memstore snapshotting TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731971770505 (+1 ms)Flushing stores of TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01. at 1731971770505Flushing 2312cbe93ba4cc288efe5635ed9f3f01/A: creating writer at 1731971770505Flushing mob file 2312cbe93ba4cc288efe5635ed9f3f01/A: appending metadata at 1731971770511 (+6 ms)Flushing mob file 2312cbe93ba4cc288efe5635ed9f3f01/A: closing flushed file at 1731971770511Flushing 2312cbe93ba4cc288efe5635ed9f3f01/A: appending metadata at 1731971770926 (+415 ms)Flushing 2312cbe93ba4cc288efe5635ed9f3f01/A: closing flushed file at 1731971770926Flushing 2312cbe93ba4cc288efe5635ed9f3f01/B: creating writer at 1731971771339 (+413 ms)Flushing 2312cbe93ba4cc288efe5635ed9f3f01/B: appending metadata at 1731971771345 (+6 ms)Flushing 2312cbe93ba4cc288efe5635ed9f3f01/B: closing flushed file at 1731971771345Flushing 2312cbe93ba4cc288efe5635ed9f3f01/C: creating writer at 1731971771370 (+25 ms)Flushing 2312cbe93ba4cc288efe5635ed9f3f01/C: appending metadata at 1731971771375 (+5 ms)Flushing 2312cbe93ba4cc288efe5635ed9f3f01/C: closing flushed file at 1731971771375Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4520a579: reopening flushed file at 1731971771792 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@640e4e91: reopening flushed file at 1731971771810 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d3e2c68: reopening flushed file at 1731971771832 (+22 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 2312cbe93ba4cc288efe5635ed9f3f01 in 1357ms, sequenceid=356, compaction requested=true at 1731971771862 (+30 ms)Writing region close event to WAL at 1731971772076 (+214 ms)Running coprocessor post-close hooks at 1731971772102 (+26 ms)Closed at 1731971772102 2024-11-18T23:16:12,120 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:12,133 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=2312cbe93ba4cc288efe5635ed9f3f01, regionState=CLOSED 2024-11-18T23:16:12,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:16:12,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-18T23:16:12,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 2312cbe93ba4cc288efe5635ed9f3f01, server=9d96596e55a9,32899,1731971713394 in 2.6360 sec 2024-11-18T23:16:12,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-18T23:16:12,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2312cbe93ba4cc288efe5635ed9f3f01, UNASSIGN in 2.6430 sec 2024-11-18T23:16:12,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-18T23:16:12,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6480 sec 2024-11-18T23:16:12,149 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971772149"}]},"ts":"1731971772149"} 2024-11-18T23:16:12,152 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T23:16:12,152 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T23:16:12,155 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6630 sec 2024-11-18T23:16:12,927 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T23:16:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-18T23:16:13,648 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:13,649 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T23:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:13,650 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:13,651 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T23:16:13,655 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,657 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/recovered.edits] 2024-11-18T23:16:13,660 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2ec2591bd03f49fbbed42ac87cf042d8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/2ec2591bd03f49fbbed42ac87cf042d8 2024-11-18T23:16:13,661 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4e3b995fcf6b483194edb5c84e2a876d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/4e3b995fcf6b483194edb5c84e2a876d 2024-11-18T23:16:13,663 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/c7e3f9371f244fde826e595b3d11ed69 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/c7e3f9371f244fde826e595b3d11ed69 2024-11-18T23:16:13,664 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e779819e7ad441e29d4593dd68ab8567 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/A/e779819e7ad441e29d4593dd68ab8567 2024-11-18T23:16:13,667 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/7ff58bdb7b0e4b108e8866d1307426e7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/7ff58bdb7b0e4b108e8866d1307426e7 2024-11-18T23:16:13,672 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ad07ebc4bc694515823d70b501b9edfb to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/ad07ebc4bc694515823d70b501b9edfb 2024-11-18T23:16:13,674 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/bd8f7ba89bd0463ab8fd05ec2370a947 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/bd8f7ba89bd0463ab8fd05ec2370a947 2024-11-18T23:16:13,676 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c7e9da3f96c141759caaa1c8e0cd0a87 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/B/c7e9da3f96c141759caaa1c8e0cd0a87 2024-11-18T23:16:13,678 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/4880512f0a3949adb17d1d98540a405e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/4880512f0a3949adb17d1d98540a405e 2024-11-18T23:16:13,679 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/5ad9772d7e3b48588d2a0418921150ba to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/5ad9772d7e3b48588d2a0418921150ba 2024-11-18T23:16:13,681 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a0eaa18d93de48c38881607799898cff to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/a0eaa18d93de48c38881607799898cff 2024-11-18T23:16:13,682 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d52dafa310074d01bff1e6c45b09de0d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/C/d52dafa310074d01bff1e6c45b09de0d 2024-11-18T23:16:13,685 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/recovered.edits/359.seqid to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01/recovered.edits/359.seqid 2024-11-18T23:16:13,685 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,685 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T23:16:13,686 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T23:16:13,686 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T23:16:13,690 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180fdb4fbbf2434d999293b32e06a49a40_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180fdb4fbbf2434d999293b32e06a49a40_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,692 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118103c7043ecc946be876abdd67ef1e12c_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118103c7043ecc946be876abdd67ef1e12c_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,693 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182771f77b329f472fa991e13885de2101_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182771f77b329f472fa991e13885de2101_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,694 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828e142d3c77b435e86715855666ff34b_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111828e142d3c77b435e86715855666ff34b_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,695 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118401e2122859d49ed9fb26ba31ee10a3c_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118401e2122859d49ed9fb26ba31ee10a3c_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,696 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184671c78c50c14d9fa3ec8618e826bc15_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184671c78c50c14d9fa3ec8618e826bc15_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,698 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111847ee34a5a27947a5b7852e61dbaa6299_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111847ee34a5a27947a5b7852e61dbaa6299_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,699 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184c38ed91a7854aadbf18c2c171d7d674_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411184c38ed91a7854aadbf18c2c171d7d674_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,701 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118533edafbf5a64c5d8c77f3ef6652527e_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118533edafbf5a64c5d8c77f3ef6652527e_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,702 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185ccfc7fd74cd4126bc6fdbc870bb6313_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185ccfc7fd74cd4126bc6fdbc870bb6313_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,703 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111861c75afbc9184b328f4f748e4cf7e641_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111861c75afbc9184b328f4f748e4cf7e641_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,704 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186475d9b15818492fb13f0a6ab889fa56_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186475d9b15818492fb13f0a6ab889fa56_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,705 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b664bc6dd1548fb8f6eeb64313d8083_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411187b664bc6dd1548fb8f6eeb64313d8083_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,707 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188f967494c184495d8b832adb5be004ed_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188f967494c184495d8b832adb5be004ed_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,708 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fabff8c2ea148c4837ff926dfd0e588_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fabff8c2ea148c4837ff926dfd0e588_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,710 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c6fbba4b52fb41daa5deba599b5858be_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118c6fbba4b52fb41daa5deba599b5858be_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,711 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118de3d494522404599b7f235de0c5652cd_2312cbe93ba4cc288efe5635ed9f3f01 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118de3d494522404599b7f235de0c5652cd_2312cbe93ba4cc288efe5635ed9f3f01 2024-11-18T23:16:13,711 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T23:16:13,714 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:13,717 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T23:16:13,719 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T23:16:13,721 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:13,721 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T23:16:13,721 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731971773721"}]},"ts":"9223372036854775807"} 2024-11-18T23:16:13,724 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T23:16:13,724 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2312cbe93ba4cc288efe5635ed9f3f01, NAME => 'TestAcidGuarantees,,1731971745115.2312cbe93ba4cc288efe5635ed9f3f01.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T23:16:13,724 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T23:16:13,724 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731971773724"}]},"ts":"9223372036854775807"} 2024-11-18T23:16:13,727 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T23:16:13,727 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:13,729 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-11-18T23:16:13,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-18T23:16:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T23:16:13,758 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:13,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 250) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1674420971_22 at /127.0.0.1:32896 [Waiting for operation #231] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1122032795_22 at /127.0.0.1:50814 [Waiting for operation #55] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1122032795_22 at /127.0.0.1:32852 [Waiting for operation #258] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1129 (was 1021) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6586 (was 6130) - AvailableMemoryMB LEAK? - 2024-11-18T23:16:13,780 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1129, ProcessCount=11, AvailableMemoryMB=6583 2024-11-18T23:16:13,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:16:13,782 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:16:13,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:13,785 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:16:13,785 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:13,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-18T23:16:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T23:16:13,786 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:16:13,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742026_1202 (size=1061) 2024-11-18T23:16:13,797 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:16:13,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742027_1203 (size=53) 2024-11-18T23:16:13,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T23:16:14,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T23:16:14,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:16:14,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d0aae39639c33cd55093c78ecbe9c4f6, disabling compactions & flushes 2024-11-18T23:16:14,207 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. after waiting 0 ms 2024-11-18T23:16:14,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,207 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,207 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0aae39639c33cd55093c78ecbe9c4f6: Waiting for close lock at 1731971774207Disabling compacts and flushes for region at 1731971774207Disabling writes for close at 1731971774207Writing region close event to WAL at 1731971774207Closed at 1731971774207 2024-11-18T23:16:14,208 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:16:14,209 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731971774208"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731971774208"}]},"ts":"1731971774208"} 2024-11-18T23:16:14,210 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:16:14,211 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:16:14,211 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971774211"}]},"ts":"1731971774211"} 2024-11-18T23:16:14,212 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T23:16:14,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, ASSIGN}] 2024-11-18T23:16:14,214 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, ASSIGN 2024-11-18T23:16:14,214 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, ASSIGN; state=OFFLINE, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=false 2024-11-18T23:16:14,365 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=d0aae39639c33cd55093c78ecbe9c4f6, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:14,366 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, ASSIGN because future has completed 2024-11-18T23:16:14,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:16:14,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T23:16:14,521 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,521 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:16:14,522 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,522 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:16:14,522 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,522 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,523 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,524 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:14,524 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0aae39639c33cd55093c78ecbe9c4f6 columnFamilyName A 2024-11-18T23:16:14,524 DEBUG [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:14,525 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.HStore(327): Store=d0aae39639c33cd55093c78ecbe9c4f6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:14,525 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,526 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:14,526 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0aae39639c33cd55093c78ecbe9c4f6 columnFamilyName B 2024-11-18T23:16:14,526 DEBUG [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:14,527 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.HStore(327): Store=d0aae39639c33cd55093c78ecbe9c4f6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:14,527 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,528 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:14,528 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0aae39639c33cd55093c78ecbe9c4f6 columnFamilyName C 2024-11-18T23:16:14,528 DEBUG [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:14,529 INFO [StoreOpener-d0aae39639c33cd55093c78ecbe9c4f6-1 {}] regionserver.HStore(327): Store=d0aae39639c33cd55093c78ecbe9c4f6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:14,529 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,529 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,529 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,530 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,531 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,531 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,531 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:16:14,532 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,534 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:16:14,535 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened d0aae39639c33cd55093c78ecbe9c4f6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66397042, jitterRate=-0.010606974363327026}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:16:14,535 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:14,535 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for d0aae39639c33cd55093c78ecbe9c4f6: Running coprocessor pre-open hook at 1731971774522Writing region info on filesystem at 1731971774522Initializing all the Stores at 1731971774522Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971774523 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971774523Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971774523Cleaning up temporary data from old regions at 1731971774531 (+8 ms)Running coprocessor post-open hooks at 1731971774535 (+4 ms)Region opened successfully at 1731971774535 2024-11-18T23:16:14,536 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., pid=59, masterSystemTime=1731971774518 2024-11-18T23:16:14,538 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,538 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:14,538 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=d0aae39639c33cd55093c78ecbe9c4f6, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:14,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:16:14,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-18T23:16:14,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 in 175 msec 2024-11-18T23:16:14,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-18T23:16:14,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, ASSIGN in 331 msec 2024-11-18T23:16:14,546 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:16:14,546 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971774546"}]},"ts":"1731971774546"} 2024-11-18T23:16:14,548 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T23:16:14,549 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:16:14,551 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 767 msec 2024-11-18T23:16:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-18T23:16:14,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:14,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a25e579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,929 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,930 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ce891a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,932 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@611da558, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,933 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48738, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:14,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:16:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,42339,1731971712494 2024-11-18T23:16:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@566a3f0c 2024-11-18T23:16:14,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:16:14,938 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:16:14,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,941 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3f2d92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f749c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,944 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,944 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,945 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d59b38d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,947 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ecba0f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ac8995, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,951 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69321540, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,953 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@399b9f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,955 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6255db1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675e61a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,958 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7191b1f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,959 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cab939, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,962 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc32559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,964 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c0b261c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5060ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9d7c6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,973 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42283f50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4ce128, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,977 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3de58d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5606186b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e1d0fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:14,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:14,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d997e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:14,983 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:14,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26f081e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:14,987 DEBUG [Thread-978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,987 DEBUG [Thread-972 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,987 DEBUG [Thread-989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,988 DEBUG [Thread-989 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,988 DEBUG [Thread-989 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,988 DEBUG [Thread-974 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:14,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:14,989 DEBUG [Thread-980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:14,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,992 DEBUG [Thread-974 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,992 DEBUG [Thread-974 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,992 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:14,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:14,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:14,994 DEBUG [Thread-985 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:14,995 DEBUG [Thread-985 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,996 DEBUG [Thread-985 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:14,997 DEBUG [Thread-987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:14,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:14,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:14,999 DEBUG [Thread-976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:15,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:15,000 DEBUG [Thread-976 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:15,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:15,002 DEBUG [Thread-976 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:15,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:15,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:15,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:15,008 DEBUG [Thread-991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:15,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:16:15,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:15,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:15,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:15,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:15,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:15,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:15,011 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:15,012 DEBUG [Thread-983 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:15,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:15,013 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:15,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:15,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:15,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-18T23:16:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T23:16:15,020 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:15,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:16:15,029 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:15,029 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:15,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/cffc58f177714eb9b97ca6a89050edd4 is 50, key is test_row_0/A:col10/1731971775008/Put/seqid=0 2024-11-18T23:16:15,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971835064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971835066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971835073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971835074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971835075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742028_1204 (size=12001) 2024-11-18T23:16:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T23:16:15,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971835187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971835187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971835187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:15,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971835199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971835208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T23:16:15,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971835397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971835398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971835397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971835408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971835417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:15,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/cffc58f177714eb9b97ca6a89050edd4 2024-11-18T23:16:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/89c072ab73964c3d828c1761dcffb6b9 is 50, key is test_row_0/B:col10/1731971775008/Put/seqid=0 2024-11-18T23:16:15,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742029_1205 (size=12001) 2024-11-18T23:16:15,583 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/89c072ab73964c3d828c1761dcffb6b9 2024-11-18T23:16:15,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/d36c21154bde438db8d9164148e2a8a8 is 50, key is test_row_0/C:col10/1731971775008/Put/seqid=0 2024-11-18T23:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T23:16:15,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742030_1206 (size=12001) 2024-11-18T23:16:15,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/d36c21154bde438db8d9164148e2a8a8 2024-11-18T23:16:15,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/cffc58f177714eb9b97ca6a89050edd4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/cffc58f177714eb9b97ca6a89050edd4 2024-11-18T23:16:15,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/cffc58f177714eb9b97ca6a89050edd4, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T23:16:15,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/89c072ab73964c3d828c1761dcffb6b9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/89c072ab73964c3d828c1761dcffb6b9 2024-11-18T23:16:15,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/89c072ab73964c3d828c1761dcffb6b9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T23:16:15,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/d36c21154bde438db8d9164148e2a8a8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d36c21154bde438db8d9164148e2a8a8 2024-11-18T23:16:15,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d36c21154bde438db8d9164148e2a8a8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-18T23:16:15,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d0aae39639c33cd55093c78ecbe9c4f6 in 700ms, sequenceid=13, compaction requested=false 2024-11-18T23:16:15,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:15,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971775069, max timestamp=1731971775712Num uniques 859; 2024-11-18T23:16:15,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971775069, max timestamp=1731971775712Num uniques 859; 2024-11-18T23:16:15,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:15,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971775069, max timestamp=1731971775712Num uniques 859; 2024-11-18T23:16:15,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:15,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:15,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-18T23:16:15,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:15,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:15,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:15,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:15,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:15,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:15,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/6428d16ffd1948f4987bc0d521b4cb6e is 50, key is test_row_0/A:col10/1731971775720/Put/seqid=0 2024-11-18T23:16:15,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742031_1207 (size=14341) 2024-11-18T23:16:15,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/6428d16ffd1948f4987bc0d521b4cb6e 2024-11-18T23:16:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971835753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971835755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971835757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971835758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971835761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/e9523efa1ef942bd8a094bab402345d2 is 50, key is test_row_0/B:col10/1731971775720/Put/seqid=0 2024-11-18T23:16:15,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742032_1208 (size=12001) 2024-11-18T23:16:15,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/e9523efa1ef942bd8a094bab402345d2 2024-11-18T23:16:15,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:15,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:15,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/04fdd87fd5474a2c8bc2a8ceedb325b6 is 50, key is test_row_0/C:col10/1731971775720/Put/seqid=0 2024-11-18T23:16:15,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742033_1209 (size=12001) 2024-11-18T23:16:15,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971835877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971835877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971835878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971835878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971835878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:15,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:15,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:15,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:15,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:15,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971836088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:16,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971836088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:16,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971836088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971836088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:16,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:16,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971836088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:16,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:16,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:16,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T23:16:16,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/04fdd87fd5474a2c8bc2a8ceedb325b6 2024-11-18T23:16:16,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/6428d16ffd1948f4987bc0d521b4cb6e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6428d16ffd1948f4987bc0d521b4cb6e 2024-11-18T23:16:16,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6428d16ffd1948f4987bc0d521b4cb6e, entries=200, sequenceid=40, filesize=14.0 K 2024-11-18T23:16:16,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/e9523efa1ef942bd8a094bab402345d2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e9523efa1ef942bd8a094bab402345d2 2024-11-18T23:16:16,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e9523efa1ef942bd8a094bab402345d2, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T23:16:16,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/04fdd87fd5474a2c8bc2a8ceedb325b6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04fdd87fd5474a2c8bc2a8ceedb325b6 2024-11-18T23:16:16,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04fdd87fd5474a2c8bc2a8ceedb325b6, entries=150, sequenceid=40, filesize=11.7 K 2024-11-18T23:16:16,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for d0aae39639c33cd55093c78ecbe9c4f6 in 580ms, sequenceid=40, compaction requested=false 2024-11-18T23:16:16,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:16,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971775750, max timestamp=1731971776400Num uniques 859; 2024-11-18T23:16:16,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:16,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971775750, max timestamp=1731971776400Num uniques 859; 2024-11-18T23:16:16,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-18T23:16:16,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971775750, max timestamp=1731971776400Num uniques 859; 2024-11-18T23:16:16,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:16,404 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:16,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:16,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/914cb2a7c57848ccbcb4900febbfab4a is 50, key is test_row_0/A:col10/1731971775756/Put/seqid=0 2024-11-18T23:16:16,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742034_1210 (size=12001) 2024-11-18T23:16:16,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:16,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971776433, max timestamp=1731971776442Num uniques 859; 2024-11-18T23:16:16,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971776433, max timestamp=1731971776442Num uniques 859; 2024-11-18T23:16:16,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:16,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:16,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971776433, max timestamp=1731971776442Num uniques 859; 2024-11-18T23:16:16,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:16,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731971776463, max timestamp=1731971776477Num uniques 1018; 2024-11-18T23:16:16,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731971776463, max timestamp=1731971776477Num uniques 1018; 2024-11-18T23:16:16,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-18T23:16:16,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-18T23:16:16,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731971776463, max timestamp=1731971776477Num uniques 1018; 2024-11-18T23:16:16,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-18T23:16:16,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971836548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971836548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:16,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:16,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731971776530, max timestamp=1731971776555Num uniques 1127; 2024-11-18T23:16:16,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731971776530, max timestamp=1731971776555Num uniques 1127; 2024-11-18T23:16:16,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-18T23:16:16,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=8102, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-18T23:16:16,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8092, totalHeapSize=18648, min timestamp=1731971776530, max timestamp=1731971776555Num uniques 1127; 2024-11-18T23:16:16,568 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51602, new segment data size=8092, suffix heap size=145176,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=177 2024-11-18T23:16:16,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731971776600, max timestamp=1731971776617Num uniques 1036; 2024-11-18T23:16:16,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T23:16:16,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731971776600, max timestamp=1731971776617Num uniques 1036; 2024-11-18T23:16:16,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T23:16:16,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731971776600, max timestamp=1731971776617Num uniques 1036; 2024-11-18T23:16:16,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-18T23:16:16,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731971776657, max timestamp=1731971776673Num uniques 1045; 2024-11-18T23:16:16,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731971776657, max timestamp=1731971776673Num uniques 1045; 2024-11-18T23:16:16,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T23:16:16,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T23:16:16,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731971776657, max timestamp=1731971776673Num uniques 1045; 2024-11-18T23:16:16,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T23:16:16,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731971776716, max timestamp=1731971776738Num uniques 1104; 2024-11-18T23:16:16,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-18T23:16:16,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971776723, max timestamp=1731971776738Num uniques 1104; 2024-11-18T23:16:16,747 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-11-18T23:16:16,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971776723, max timestamp=1731971776738Num uniques 1104; 2024-11-18T23:16:16,748 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=6870, suffix heap size=142784,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=150 2024-11-18T23:16:16,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971776770, max timestamp=1731971776789Num uniques 1009; 2024-11-18T23:16:16,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:16,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971776770, max timestamp=1731971776789Num uniques 1009; 2024-11-18T23:16:16,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971776770, max timestamp=1731971776789Num uniques 1063; 2024-11-18T23:16:16,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:16,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=6870, suffix heap size=138520,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=150 2024-11-18T23:16:16,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/914cb2a7c57848ccbcb4900febbfab4a 2024-11-18T23:16:16,845 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731971776825, max timestamp=1731971776837Num uniques 1009; 2024-11-18T23:16:16,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7872, suffix heap size=132904,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=172 2024-11-18T23:16:16,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731971776825, max timestamp=1731971776837Num uniques 1009; 2024-11-18T23:16:16,846 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7872, suffix heap size=132904,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=172 2024-11-18T23:16:16,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731971776825, max timestamp=1731971776837Num uniques 1009; 2024-11-18T23:16:16,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7872, suffix heap size=132904,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=172 2024-11-18T23:16:16,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/c86f371e4c1c49229ac3138717444905 is 50, key is test_row_0/B:col10/1731971775756/Put/seqid=0 2024-11-18T23:16:16,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731971776863, max timestamp=1731971776876Num uniques 1031; 2024-11-18T23:16:16,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=12866, suffix heap size=135192,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=281 2024-11-18T23:16:16,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742035_1211 (size=12001) 2024-11-18T23:16:16,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731971776871, max timestamp=1731971776876Num uniques 1031; 2024-11-18T23:16:16,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-18T23:16:16,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731971776871, max timestamp=1731971776876Num uniques 1031; 2024-11-18T23:16:16,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-18T23:16:16,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/c86f371e4c1c49229ac3138717444905 2024-11-18T23:16:16,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:16,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:16,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:16,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ded4d638590049faae7c7d7cc945352b is 50, key is test_row_0/C:col10/1731971775756/Put/seqid=0 2024-11-18T23:16:16,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731971776923, max timestamp=1731971776936Num uniques 1040; 2024-11-18T23:16:16,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731971776923, max timestamp=1731971776936Num uniques 1140; 2024-11-18T23:16:16,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-18T23:16:16,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-18T23:16:16,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=200, cellSize=9160, totalHeapSize=21040, min timestamp=1731971776923, max timestamp=1731971776936Num uniques 1040; 2024-11-18T23:16:16,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=9160, suffix heap size=136128,new segment heap size=21040 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=200 2024-11-18T23:16:16,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742036_1212 (size=12001) 2024-11-18T23:16:17,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731971776982, max timestamp=1731971777006Num uniques 1099; 2024-11-18T23:16:17,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-18T23:16:17,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731971776982, max timestamp=1731971777006Num uniques 1059; 2024-11-18T23:16:17,015 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48504, new segment data size=11404, suffix heap size=138104,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1059, new segment cells count=249 2024-11-18T23:16:17,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731971776982, max timestamp=1731971777006Num uniques 1099; 2024-11-18T23:16:17,019 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-18T23:16:17,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9518, totalHeapSize=21872, min timestamp=1731971777053, max timestamp=1731971777070Num uniques 1108; 2024-11-18T23:16:17,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50738, new segment data size=9518, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-18T23:16:17,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731971777053, max timestamp=1731971777070Num uniques 1108; 2024-11-18T23:16:17,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=9520, suffix heap size=143200,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=208 2024-11-18T23:16:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777053, max timestamp=1731971777070Num uniques 1108; 2024-11-18T23:16:17,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=6870, suffix heap size=143200,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=150 2024-11-18T23:16:17,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:17,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:17,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731971777104, max timestamp=1731971777119Num uniques 1067; 2024-11-18T23:16:17,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-18T23:16:17,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731971777104, max timestamp=1731971777119Num uniques 1067; 2024-11-18T23:16:17,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-18T23:16:17,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731971777104, max timestamp=1731971777119Num uniques 1009; 2024-11-18T23:16:17,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46212, new segment data size=7642, suffix heap size=132904,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=167 2024-11-18T23:16:17,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T23:16:17,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731971777180, max timestamp=1731971777198Num uniques 1026; 2024-11-18T23:16:17,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-11-18T23:16:17,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777180, max timestamp=1731971777198Num uniques 1026; 2024-11-18T23:16:17,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=6870, suffix heap size=134672,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=150 2024-11-18T23:16:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777180, max timestamp=1731971777198Num uniques 1026; 2024-11-18T23:16:17,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=6870, suffix heap size=134672,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=150 2024-11-18T23:16:17,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731971777240, max timestamp=1731971777257Num uniques 1085; 2024-11-18T23:16:17,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=13050, suffix heap size=140808,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=285 2024-11-18T23:16:17,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731971777240, max timestamp=1731971777257Num uniques 1009; 2024-11-18T23:16:17,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13050, suffix heap size=132904,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=285 2024-11-18T23:16:17,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=285, cellSize=13050, totalHeapSize=29880, min timestamp=1731971777240, max timestamp=1731971777257Num uniques 1009; 2024-11-18T23:16:17,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=13050, suffix heap size=132904,new segment heap size=29880 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=285 2024-11-18T23:16:17,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777294, max timestamp=1731971777311Num uniques 1144; 2024-11-18T23:16:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777294, max timestamp=1731971777311Num uniques 1144; 2024-11-18T23:16:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777294, max timestamp=1731971777311Num uniques 1144; 2024-11-18T23:16:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=6870, suffix heap size=146944,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=150 2024-11-18T23:16:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=6870, suffix heap size=146944,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=150 2024-11-18T23:16:17,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52394, new segment data size=6870, suffix heap size=146944,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1144, new segment cells count=150 2024-11-18T23:16:17,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ded4d638590049faae7c7d7cc945352b 2024-11-18T23:16:17,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/914cb2a7c57848ccbcb4900febbfab4a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/914cb2a7c57848ccbcb4900febbfab4a 2024-11-18T23:16:17,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=253, cellSize=11585, totalHeapSize=26552, min timestamp=1731971777357, max timestamp=1731971777386Num uniques 1009; 2024-11-18T23:16:17,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=11585, suffix heap size=132904,new segment heap size=26552 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=253 2024-11-18T23:16:17,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777357, max timestamp=1731971777390Num uniques 1009; 2024-11-18T23:16:17,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:17,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971777357, max timestamp=1731971777390Num uniques 1009; 2024-11-18T23:16:17,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:17,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,406 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/914cb2a7c57848ccbcb4900febbfab4a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:16:17,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/c86f371e4c1c49229ac3138717444905 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c86f371e4c1c49229ac3138717444905 2024-11-18T23:16:17,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c86f371e4c1c49229ac3138717444905, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:16:17,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ded4d638590049faae7c7d7cc945352b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ded4d638590049faae7c7d7cc945352b 2024-11-18T23:16:17,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731971777431, max timestamp=1731971777451Num uniques 1112; 2024-11-18T23:16:17,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731971777431, max timestamp=1731971777451Num uniques 1009; 2024-11-18T23:16:17,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50922, new segment data size=7412, suffix heap size=143616,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1112, new segment cells count=162 2024-11-18T23:16:17,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=7412, suffix heap size=132904,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=162 2024-11-18T23:16:17,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731971777431, max timestamp=1731971777451Num uniques 1009; 2024-11-18T23:16:17,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46207, new segment data size=7412, suffix heap size=132904,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=162 2024-11-18T23:16:17,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ded4d638590049faae7c7d7cc945352b, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:16:17,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=40.25 KB/41220 for d0aae39639c33cd55093c78ecbe9c4f6 in 1057ms, sequenceid=61, compaction requested=true 2024-11-18T23:16:17,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:17,462 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:17,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:17,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:17,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:17,462 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:17,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:17,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:17,464 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:17,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:17,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=60.38 KB heapSize=165.56 KB 2024-11-18T23:16:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:17,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:17,477 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:17,477 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:17,477 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,478 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/cffc58f177714eb9b97ca6a89050edd4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6428d16ffd1948f4987bc0d521b4cb6e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/914cb2a7c57848ccbcb4900febbfab4a] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=37.4 K 2024-11-18T23:16:17,488 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting cffc58f177714eb9b97ca6a89050edd4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731971775007 2024-11-18T23:16:17,492 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:17,492 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:17,492 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,493 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/89c072ab73964c3d828c1761dcffb6b9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e9523efa1ef942bd8a094bab402345d2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c86f371e4c1c49229ac3138717444905] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=35.2 K 2024-11-18T23:16:17,493 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6428d16ffd1948f4987bc0d521b4cb6e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731971775069 2024-11-18T23:16:17,493 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c072ab73964c3d828c1761dcffb6b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731971775007 2024-11-18T23:16:17,496 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 914cb2a7c57848ccbcb4900febbfab4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971775756 2024-11-18T23:16:17,498 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9523efa1ef942bd8a094bab402345d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731971775720 2024-11-18T23:16:17,504 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86f371e4c1c49229ac3138717444905, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971775756 2024-11-18T23:16:17,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/f521e8f6026b4e8184cfea8a4dc33dd9 is 50, key is test_row_0/A:col10/1731971777470/Put/seqid=0 2024-11-18T23:16:17,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731971777511, max timestamp=1731971777539Num uniques 859; 2024-11-18T23:16:17,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-18T23:16:17,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742037_1213 (size=19621) 2024-11-18T23:16:17,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971777530, max timestamp=1731971777539Num uniques 859; 2024-11-18T23:16:17,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971777530, max timestamp=1731971777539Num uniques 859; 2024-11-18T23:16:17,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:17,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:17,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/f521e8f6026b4e8184cfea8a4dc33dd9 2024-11-18T23:16:17,560 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:17,560 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/9515bbc9abec458d8f229332b54df9a8 is 50, key is test_row_0/A:col10/1731971775756/Put/seqid=0 2024-11-18T23:16:17,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,567 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:17,567 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/7747544eebc947969162c895dbd865e3 is 50, key is test_row_0/B:col10/1731971775756/Put/seqid=0 2024-11-18T23:16:17,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742038_1214 (size=12104) 2024-11-18T23:16:17,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971837588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/737a746a1ac04648b233954736fa63d1 is 50, key is test_row_0/B:col10/1731971777470/Put/seqid=0 2024-11-18T23:16:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971837591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,607 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971837592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971837595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971837599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742039_1215 (size=12104) 2024-11-18T23:16:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,662 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:16:17,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742040_1216 (size=12301) 2024-11-18T23:16:17,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971837712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971837718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971837719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971837721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971837741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:17,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:17,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:17,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971837940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971837940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,945 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971837947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971837947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971837960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:17,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,026 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/9515bbc9abec458d8f229332b54df9a8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9515bbc9abec458d8f229332b54df9a8 2024-11-18T23:16:18,037 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/7747544eebc947969162c895dbd865e3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7747544eebc947969162c895dbd865e3 2024-11-18T23:16:18,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,057 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into 9515bbc9abec458d8f229332b54df9a8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:18,057 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:18,057 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971777462; duration=0sec 2024-11-18T23:16:18,057 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:18,057 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:18,057 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:18,065 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into 7747544eebc947969162c895dbd865e3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:18,065 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:18,065 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971777462; duration=0sec 2024-11-18T23:16:18,065 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:18,065 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:18,065 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:18,065 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:18,065 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,066 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d36c21154bde438db8d9164148e2a8a8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04fdd87fd5474a2c8bc2a8ceedb325b6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ded4d638590049faae7c7d7cc945352b] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=35.2 K 2024-11-18T23:16:18,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/737a746a1ac04648b233954736fa63d1 2024-11-18T23:16:18,067 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d36c21154bde438db8d9164148e2a8a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731971775007 2024-11-18T23:16:18,068 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 04fdd87fd5474a2c8bc2a8ceedb325b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731971775720 2024-11-18T23:16:18,068 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ded4d638590049faae7c7d7cc945352b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971775756 2024-11-18T23:16:18,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b1c886f278a6440ca2b3b9b5c0c5715e is 50, key is test_row_0/C:col10/1731971777470/Put/seqid=0 2024-11-18T23:16:18,102 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#178 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:18,103 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b9c9ecb131674b589ec31e6c4e7cd5ef is 50, key is test_row_0/C:col10/1731971775756/Put/seqid=0 2024-11-18T23:16:18,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742041_1217 (size=12301) 2024-11-18T23:16:18,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742042_1218 (size=12104) 2024-11-18T23:16:18,126 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b9c9ecb131674b589ec31e6c4e7cd5ef as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b9c9ecb131674b589ec31e6c4e7cd5ef 2024-11-18T23:16:18,141 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into b9c9ecb131674b589ec31e6c4e7cd5ef(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:18,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:18,141 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971777462; duration=0sec 2024-11-18T23:16:18,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:18,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:18,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971838248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971838261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971838262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971838264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,276 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971838283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,382 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:18,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:18,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b1c886f278a6440ca2b3b9b5c0c5715e 2024-11-18T23:16:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/f521e8f6026b4e8184cfea8a4dc33dd9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f521e8f6026b4e8184cfea8a4dc33dd9 2024-11-18T23:16:18,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:18,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f521e8f6026b4e8184cfea8a4dc33dd9, entries=300, sequenceid=379, filesize=19.2 K 2024-11-18T23:16:18,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/737a746a1ac04648b233954736fa63d1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/737a746a1ac04648b233954736fa63d1 2024-11-18T23:16:18,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/737a746a1ac04648b233954736fa63d1, entries=150, sequenceid=379, filesize=12.0 K 2024-11-18T23:16:18,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b1c886f278a6440ca2b3b9b5c0c5715e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b1c886f278a6440ca2b3b9b5c0c5715e 2024-11-18T23:16:18,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b1c886f278a6440ca2b3b9b5c0c5715e, entries=150, sequenceid=379, filesize=12.0 K 2024-11-18T23:16:18,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.52 KB/169488, currentSize=129.71 KB/132820 for d0aae39639c33cd55093c78ecbe9c4f6 in 1115ms, sequenceid=379, compaction requested=false 2024-11-18T23:16:18,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:18,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-18T23:16:18,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,704 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=129.71 KB heapSize=365.45 KB 2024-11-18T23:16:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:18,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:18,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/df87b5504ab2419bb1043163875828c7 is 50, key is test_row_0/A:col10/1731971777593/Put/seqid=0 2024-11-18T23:16:18,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742043_1219 (size=12301) 2024-11-18T23:16:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:18,768 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/df87b5504ab2419bb1043163875828c7 2024-11-18T23:16:18,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/5aae61d7135f4b96882b812110962279 is 50, key is test_row_0/B:col10/1731971777593/Put/seqid=0 2024-11-18T23:16:18,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742044_1220 (size=12301) 2024-11-18T23:16:18,804 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/5aae61d7135f4b96882b812110962279 2024-11-18T23:16:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971838802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971838802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971838806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/c3fa4b00cd544c7eb191f24b72287b70 is 50, key is test_row_0/C:col10/1731971777593/Put/seqid=0 2024-11-18T23:16:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971838810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971838814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:18,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:18,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742045_1221 (size=12301) 2024-11-18T23:16:18,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/c3fa4b00cd544c7eb191f24b72287b70 2024-11-18T23:16:18,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/df87b5504ab2419bb1043163875828c7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/df87b5504ab2419bb1043163875828c7 2024-11-18T23:16:18,871 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/df87b5504ab2419bb1043163875828c7, entries=150, sequenceid=417, filesize=12.0 K 2024-11-18T23:16:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/5aae61d7135f4b96882b812110962279 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/5aae61d7135f4b96882b812110962279 2024-11-18T23:16:18,886 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/5aae61d7135f4b96882b812110962279, entries=150, sequenceid=417, filesize=12.0 K 2024-11-18T23:16:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/c3fa4b00cd544c7eb191f24b72287b70 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c3fa4b00cd544c7eb191f24b72287b70 2024-11-18T23:16:18,907 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c3fa4b00cd544c7eb191f24b72287b70, entries=150, sequenceid=417, filesize=12.0 K 2024-11-18T23:16:18,909 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~365.41 KB/374176, currentSize=53.67 KB/54960 for d0aae39639c33cd55093c78ecbe9c4f6 in 205ms, sequenceid=417, compaction requested=true 2024-11-18T23:16:18,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:18,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:18,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-18T23:16:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-18T23:16:18,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-18T23:16:18,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8870 sec 2024-11-18T23:16:18,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 3.9060 sec 2024-11-18T23:16:18,936 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T23:16:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:18,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:18,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/d4f275c7344f40ef829a9a6765f94837 is 50, key is test_row_0/A:col10/1731971778931/Put/seqid=0 2024-11-18T23:16:18,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742046_1222 (size=12301) 2024-11-18T23:16:18,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/d4f275c7344f40ef829a9a6765f94837 2024-11-18T23:16:19,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/20d70f64723c465fbd5debdc04e0dd82 is 50, key is test_row_0/B:col10/1731971778931/Put/seqid=0 2024-11-18T23:16:19,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971839019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971839023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971839024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971839024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742047_1223 (size=12301) 2024-11-18T23:16:19,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/20d70f64723c465fbd5debdc04e0dd82 2024-11-18T23:16:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/efdfe68741aa41e28fb5e7a4bb0db4c5 is 50, key is test_row_0/C:col10/1731971778931/Put/seqid=0 2024-11-18T23:16:19,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742048_1224 (size=12301) 2024-11-18T23:16:19,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/efdfe68741aa41e28fb5e7a4bb0db4c5 2024-11-18T23:16:19,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/d4f275c7344f40ef829a9a6765f94837 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d4f275c7344f40ef829a9a6765f94837 2024-11-18T23:16:19,146 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d4f275c7344f40ef829a9a6765f94837, entries=150, sequenceid=431, filesize=12.0 K 2024-11-18T23:16:19,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/20d70f64723c465fbd5debdc04e0dd82 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/20d70f64723c465fbd5debdc04e0dd82 2024-11-18T23:16:19,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971839138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/20d70f64723c465fbd5debdc04e0dd82, entries=150, sequenceid=431, filesize=12.0 K 2024-11-18T23:16:19,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/efdfe68741aa41e28fb5e7a4bb0db4c5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/efdfe68741aa41e28fb5e7a4bb0db4c5 2024-11-18T23:16:19,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/efdfe68741aa41e28fb5e7a4bb0db4c5, entries=150, sequenceid=431, filesize=12.0 K 2024-11-18T23:16:19,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for d0aae39639c33cd55093c78ecbe9c4f6 in 239ms, sequenceid=431, compaction requested=true 2024-11-18T23:16:19,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:19,176 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:19,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:19,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:19,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:19,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:19,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:19,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:19,179 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56327 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:19,179 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:19,179 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,180 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9515bbc9abec458d8f229332b54df9a8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f521e8f6026b4e8184cfea8a4dc33dd9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/df87b5504ab2419bb1043163875828c7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d4f275c7344f40ef829a9a6765f94837] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=55.0 K 2024-11-18T23:16:19,180 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:19,180 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9515bbc9abec458d8f229332b54df9a8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971775756 2024-11-18T23:16:19,180 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f521e8f6026b4e8184cfea8a4dc33dd9, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731971777448 2024-11-18T23:16:19,181 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting df87b5504ab2419bb1043163875828c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731971777591 2024-11-18T23:16:19,181 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d4f275c7344f40ef829a9a6765f94837, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731971778931 2024-11-18T23:16:19,196 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49007 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:19,196 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:19,196 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,196 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7747544eebc947969162c895dbd865e3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/737a746a1ac04648b233954736fa63d1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/5aae61d7135f4b96882b812110962279, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/20d70f64723c465fbd5debdc04e0dd82] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=47.9 K 2024-11-18T23:16:19,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971778969, max timestamp=1731971779194Num uniques 859; 2024-11-18T23:16:19,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:19,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971778969, max timestamp=1731971779194Num uniques 859; 2024-11-18T23:16:19,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:19,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=64.85 KB heapSize=184.09 KB 2024-11-18T23:16:19,200 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7747544eebc947969162c895dbd865e3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971775756 2024-11-18T23:16:19,201 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:19,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971778969, max timestamp=1731971779194Num uniques 859; 2024-11-18T23:16:19,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:16:19,202 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/c79defc0577f457e9fe0a81b21cf64f9 is 50, key is test_row_0/A:col10/1731971778931/Put/seqid=0 2024-11-18T23:16:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-18T23:16:19,202 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:19,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:19,203 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:19,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:19,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:19,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:19,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:19,203 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-18T23:16:19,208 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T23:16:19,208 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 737a746a1ac04648b233954736fa63d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731971777454 2024-11-18T23:16:19,209 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:19,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:19,209 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aae61d7135f4b96882b812110962279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731971777591 2024-11-18T23:16:19,209 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20d70f64723c465fbd5debdc04e0dd82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731971778931 2024-11-18T23:16:19,246 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#186 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:19,247 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/7c753b7a02f54f3c90a69d2c2b194ce8 is 50, key is test_row_0/B:col10/1731971778931/Put/seqid=0 2024-11-18T23:16:19,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742049_1225 (size=12541) 2024-11-18T23:16:19,272 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/c79defc0577f457e9fe0a81b21cf64f9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/c79defc0577f457e9fe0a81b21cf64f9 2024-11-18T23:16:19,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/87f2f410db7f46a39a363d47044e86f5 is 50, key is test_row_0/A:col10/1731971779196/Put/seqid=0 2024-11-18T23:16:19,280 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into c79defc0577f457e9fe0a81b21cf64f9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:19,280 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:19,280 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=12, startTime=1731971779176; duration=0sec 2024-11-18T23:16:19,280 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:19,280 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:19,280 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:19,282 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49007 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:19,282 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:19,282 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,282 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b9c9ecb131674b589ec31e6c4e7cd5ef, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b1c886f278a6440ca2b3b9b5c0c5715e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c3fa4b00cd544c7eb191f24b72287b70, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/efdfe68741aa41e28fb5e7a4bb0db4c5] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=47.9 K 2024-11-18T23:16:19,283 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting b9c9ecb131674b589ec31e6c4e7cd5ef, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971775756 2024-11-18T23:16:19,284 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting b1c886f278a6440ca2b3b9b5c0c5715e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731971777454 2024-11-18T23:16:19,289 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c3fa4b00cd544c7eb191f24b72287b70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731971777591 2024-11-18T23:16:19,290 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting efdfe68741aa41e28fb5e7a4bb0db4c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731971778931 2024-11-18T23:16:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T23:16:19,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742051_1227 (size=14741) 2024-11-18T23:16:19,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/87f2f410db7f46a39a363d47044e86f5 2024-11-18T23:16:19,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971779283, max timestamp=1731971779322Num uniques 859; 2024-11-18T23:16:19,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971779283, max timestamp=1731971779322Num uniques 859; 2024-11-18T23:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:19,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971779283, max timestamp=1731971779322Num uniques 859; 2024-11-18T23:16:19,339 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:19,343 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#188 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:19,343 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/622706c549f148a9890341223175deb4 is 50, key is test_row_0/C:col10/1731971778931/Put/seqid=0 2024-11-18T23:16:19,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742050_1226 (size=12541) 2024-11-18T23:16:19,368 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/7c753b7a02f54f3c90a69d2c2b194ce8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7c753b7a02f54f3c90a69d2c2b194ce8 2024-11-18T23:16:19,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:19,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:19,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,385 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into 7c753b7a02f54f3c90a69d2c2b194ce8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:19,385 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:19,385 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=12, startTime=1731971779176; duration=0sec 2024-11-18T23:16:19,385 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:19,385 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:19,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/3176ed32b5fa41fd950780c1a03edbf4 is 50, key is test_row_0/B:col10/1731971779196/Put/seqid=0 2024-11-18T23:16:19,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742052_1228 (size=12541) 2024-11-18T23:16:19,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971839440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731971779418, max timestamp=1731971779442Num uniques 1018; 2024-11-18T23:16:19,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T23:16:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731971779418, max timestamp=1731971779442Num uniques 1018; 2024-11-18T23:16:19,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T23:16:19,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742053_1229 (size=12301) 2024-11-18T23:16:19,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731971779418, max timestamp=1731971779442Num uniques 1018; 2024-11-18T23:16:19,468 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T23:16:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T23:16:19,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971779503, max timestamp=1731971779534Num uniques 1027; 2024-11-18T23:16:19,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-18T23:16:19,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971779503, max timestamp=1731971779534Num uniques 1027; 2024-11-18T23:16:19,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971779503, max timestamp=1731971779534Num uniques 1027; 2024-11-18T23:16:19,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-18T23:16:19,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-18T23:16:19,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731971779579, max timestamp=1731971779624Num uniques 1086; 2024-11-18T23:16:19,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-18T23:16:19,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731971779579, max timestamp=1731971779624Num uniques 1086; 2024-11-18T23:16:19,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-18T23:16:19,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731971779579, max timestamp=1731971779624Num uniques 1086; 2024-11-18T23:16:19,637 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-18T23:16:19,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:19,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,686 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971839695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731971779664, max timestamp=1731971779698Num uniques 1095; 2024-11-18T23:16:19,706 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-18T23:16:19,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971779664, max timestamp=1731971779698Num uniques 1095; 2024-11-18T23:16:19,707 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-11-18T23:16:19,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971779664, max timestamp=1731971779698Num uniques 1095; 2024-11-18T23:16:19,710 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=6870, suffix heap size=141848,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=150 2024-11-18T23:16:19,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971839764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731971779760, max timestamp=1731971779770Num uniques 1054; 2024-11-18T23:16:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:19,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-18T23:16:19,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971839772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:19,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731971779760, max timestamp=1731971779770Num uniques 1009; 2024-11-18T23:16:19,781 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=7050, suffix heap size=132904,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=154 2024-11-18T23:16:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:19,781 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:19,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971779760, max timestamp=1731971779770Num uniques 1009; 2024-11-18T23:16:19,783 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:19,823 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/622706c549f148a9890341223175deb4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/622706c549f148a9890341223175deb4 2024-11-18T23:16:19,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:19,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:19,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T23:16:19,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/3176ed32b5fa41fd950780c1a03edbf4 2024-11-18T23:16:19,861 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into 622706c549f148a9890341223175deb4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:19,861 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:19,861 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=12, startTime=1731971779176; duration=0sec 2024-11-18T23:16:19,861 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:19,861 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:19,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b92fa8e84f934fe4ae36ee2fe150ff3a is 50, key is test_row_0/C:col10/1731971779196/Put/seqid=0 2024-11-18T23:16:19,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731971779858, max timestamp=1731971779913Num uniques 1013; 2024-11-18T23:16:19,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=14328, suffix heap size=133320,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=313 2024-11-18T23:16:19,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731971779858, max timestamp=1731971779913Num uniques 1009; 2024-11-18T23:16:19,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=14328, suffix heap size=132904,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=313 2024-11-18T23:16:19,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=313, cellSize=14328, totalHeapSize=32792, min timestamp=1731971779858, max timestamp=1731971779913Num uniques 1013; 2024-11-18T23:16:19,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=14328, suffix heap size=133320,new segment heap size=32792 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=313 2024-11-18T23:16:19,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742054_1230 (size=12301) 2024-11-18T23:16:19,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=454 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b92fa8e84f934fe4ae36ee2fe150ff3a 2024-11-18T23:16:19,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:19,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:19,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:19,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/87f2f410db7f46a39a363d47044e86f5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/87f2f410db7f46a39a363d47044e86f5 2024-11-18T23:16:20,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971839982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971839985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971839988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971839994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971840001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/87f2f410db7f46a39a363d47044e86f5, entries=200, sequenceid=454, filesize=14.4 K 2024-11-18T23:16:20,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/3176ed32b5fa41fd950780c1a03edbf4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/3176ed32b5fa41fd950780c1a03edbf4 2024-11-18T23:16:20,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/3176ed32b5fa41fd950780c1a03edbf4, entries=150, sequenceid=454, filesize=12.0 K 2024-11-18T23:16:20,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b92fa8e84f934fe4ae36ee2fe150ff3a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b92fa8e84f934fe4ae36ee2fe150ff3a 2024-11-18T23:16:20,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b92fa8e84f934fe4ae36ee2fe150ff3a, entries=150, sequenceid=454, filesize=12.0 K 2024-11-18T23:16:20,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.86 KB/40815, heapSize ~105.12 KB/107640, currentSize=147.60 KB/151140 for d0aae39639c33cd55093c78ecbe9c4f6 in 905ms, sequenceid=454, compaction requested=false 2024-11-18T23:16:20,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:20,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:20,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=161.02 KB heapSize=450.94 KB 2024-11-18T23:16:20,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731971779972, max timestamp=1731971780132Num uniques 1172; 2024-11-18T23:16:20,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-18T23:16:20,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731971779972, max timestamp=1731971780132Num uniques 1172; 2024-11-18T23:16:20,142 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53672, new segment data size=10162, suffix heap size=149856,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1172, new segment cells count=222 2024-11-18T23:16:20,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:20,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:20,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:20,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:20,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:20,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:16:20,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:20,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:20,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/6946515d028742d0b3107f3ecb3592a7 is 50, key is test_row_0/A:col10/1731971780136/Put/seqid=0 2024-11-18T23:16:20,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742055_1231 (size=12301) 2024-11-18T23:16:20,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971840219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971840223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/6946515d028742d0b3107f3ecb3592a7 2024-11-18T23:16:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971840224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971840231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971840231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/dd47ce58ca494922ba15828448f4864f is 50, key is test_row_0/B:col10/1731971780136/Put/seqid=0 2024-11-18T23:16:20,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742056_1232 (size=12301) 2024-11-18T23:16:20,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971840349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971840353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T23:16:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971840356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971840356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971840358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971840568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971840569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971840577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971840578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971840580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/dd47ce58ca494922ba15828448f4864f 2024-11-18T23:16:20,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/cd4eb7978d9647edac99067dfba0f08c is 50, key is test_row_0/C:col10/1731971780136/Put/seqid=0 2024-11-18T23:16:20,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742057_1233 (size=12301) 2024-11-18T23:16:20,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:20,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971840880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971840887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,890 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971840887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971840887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971840900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:20,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:20,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:20,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:21,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:21,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:21,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:21,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:21,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:21,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:21,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/cd4eb7978d9647edac99067dfba0f08c 2024-11-18T23:16:21,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/6946515d028742d0b3107f3ecb3592a7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6946515d028742d0b3107f3ecb3592a7 2024-11-18T23:16:21,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6946515d028742d0b3107f3ecb3592a7, entries=150, sequenceid=599, filesize=12.0 K 2024-11-18T23:16:21,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/dd47ce58ca494922ba15828448f4864f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dd47ce58ca494922ba15828448f4864f 2024-11-18T23:16:21,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dd47ce58ca494922ba15828448f4864f, entries=150, sequenceid=599, filesize=12.0 K 2024-11-18T23:16:21,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/cd4eb7978d9647edac99067dfba0f08c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/cd4eb7978d9647edac99067dfba0f08c 2024-11-18T23:16:21,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/cd4eb7978d9647edac99067dfba0f08c, entries=150, sequenceid=599, filesize=12.0 K 2024-11-18T23:16:21,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-18T23:16:21,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~82.74 KB/84730, heapSize ~223.69 KB/229056, currentSize=100.63 KB/103050 for d0aae39639c33cd55093c78ecbe9c4f6 in 1159ms, sequenceid=599, compaction requested=true 2024-11-18T23:16:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:21,300 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:21,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:21,300 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T23:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:21,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:21,301 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:21,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39583 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:21,306 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:21,306 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:21,306 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/c79defc0577f457e9fe0a81b21cf64f9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/87f2f410db7f46a39a363d47044e86f5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6946515d028742d0b3107f3ecb3592a7] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=38.7 K 2024-11-18T23:16:21,307 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting c79defc0577f457e9fe0a81b21cf64f9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731971778931 2024-11-18T23:16:21,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:21,319 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87f2f410db7f46a39a363d47044e86f5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731971778969 2024-11-18T23:16:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/33cb6e71393b4388af2f7b5dc08ce500 is 50, key is test_row_0/A:col10/1731971780223/Put/seqid=0 2024-11-18T23:16:21,321 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37143 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:21,321 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6946515d028742d0b3107f3ecb3592a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731971780128 2024-11-18T23:16:21,321 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:21,321 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:21,321 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7c753b7a02f54f3c90a69d2c2b194ce8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/3176ed32b5fa41fd950780c1a03edbf4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dd47ce58ca494922ba15828448f4864f] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.3 K 2024-11-18T23:16:21,328 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c753b7a02f54f3c90a69d2c2b194ce8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731971778931 2024-11-18T23:16:21,333 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3176ed32b5fa41fd950780c1a03edbf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731971778969 2024-11-18T23:16:21,339 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting dd47ce58ca494922ba15828448f4864f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731971780128 2024-11-18T23:16:21,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742058_1234 (size=12301) 2024-11-18T23:16:21,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T23:16:21,381 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/33cb6e71393b4388af2f7b5dc08ce500 2024-11-18T23:16:21,391 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:21,392 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/63de77569f0440bebd9994871035adfd is 50, key is test_row_0/A:col10/1731971780136/Put/seqid=0 2024-11-18T23:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:21,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:21,423 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:21,423 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/f767bde680d246ff94ee0d9484092976 is 50, key is test_row_0/B:col10/1731971780136/Put/seqid=0 2024-11-18T23:16:21,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/f750e4da358e4df3a785deb1f6c928b8 is 50, key is test_row_0/B:col10/1731971780223/Put/seqid=0 2024-11-18T23:16:21,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742059_1235 (size=12643) 2024-11-18T23:16:21,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971841455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971841457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971841465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971841466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971841468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,484 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/63de77569f0440bebd9994871035adfd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/63de77569f0440bebd9994871035adfd 2024-11-18T23:16:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742060_1236 (size=12643) 2024-11-18T23:16:21,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742061_1237 (size=12301) 2024-11-18T23:16:21,510 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into 63de77569f0440bebd9994871035adfd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:21,510 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:21,510 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971781300; duration=0sec 2024-11-18T23:16:21,510 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:21,510 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:21,510 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:21,521 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37143 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:21,521 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:21,521 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:21,521 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/622706c549f148a9890341223175deb4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b92fa8e84f934fe4ae36ee2fe150ff3a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/cd4eb7978d9647edac99067dfba0f08c] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.3 K 2024-11-18T23:16:21,523 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622706c549f148a9890341223175deb4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1731971778931 2024-11-18T23:16:21,524 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/f750e4da358e4df3a785deb1f6c928b8 2024-11-18T23:16:21,525 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b92fa8e84f934fe4ae36ee2fe150ff3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=454, earliestPutTs=1731971778969 2024-11-18T23:16:21,525 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd4eb7978d9647edac99067dfba0f08c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731971780128 2024-11-18T23:16:21,534 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:21,535 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ebaea5f1f7034671bdd03f1923765b1f is 50, key is test_row_0/C:col10/1731971780136/Put/seqid=0 2024-11-18T23:16:21,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742062_1238 (size=12643) 2024-11-18T23:16:21,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/e290962ae96d41f785879936d7195d81 is 50, key is test_row_0/C:col10/1731971780223/Put/seqid=0 2024-11-18T23:16:21,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971841588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971841589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971841589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,606 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ebaea5f1f7034671bdd03f1923765b1f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ebaea5f1f7034671bdd03f1923765b1f 2024-11-18T23:16:21,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971841598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971841598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742063_1239 (size=12301) 2024-11-18T23:16:21,616 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into ebaea5f1f7034671bdd03f1923765b1f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:21,616 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:21,616 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971781300; duration=0sec 2024-11-18T23:16:21,616 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:21,616 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:21,628 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/e290962ae96d41f785879936d7195d81 2024-11-18T23:16:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/33cb6e71393b4388af2f7b5dc08ce500 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/33cb6e71393b4388af2f7b5dc08ce500 2024-11-18T23:16:21,694 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/33cb6e71393b4388af2f7b5dc08ce500, entries=150, sequenceid=617, filesize=12.0 K 2024-11-18T23:16:21,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/f750e4da358e4df3a785deb1f6c928b8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f750e4da358e4df3a785deb1f6c928b8 2024-11-18T23:16:21,722 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f750e4da358e4df3a785deb1f6c928b8, entries=150, sequenceid=617, filesize=12.0 K 2024-11-18T23:16:21,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/e290962ae96d41f785879936d7195d81 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/e290962ae96d41f785879936d7195d81 2024-11-18T23:16:21,773 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/e290962ae96d41f785879936d7195d81, entries=150, sequenceid=617, filesize=12.0 K 2024-11-18T23:16:21,778 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for d0aae39639c33cd55093c78ecbe9c4f6 in 477ms, sequenceid=617, compaction requested=false 2024-11-18T23:16:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:21,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-18T23:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-18T23:16:21,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-18T23:16:21,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5710 sec 2024-11-18T23:16:21,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.5800 sec 2024-11-18T23:16:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:21,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T23:16:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:21,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:21,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/5c4afd79f3a14b678e14497e0bd6d24a is 50, key is test_row_0/A:col10/1731971781439/Put/seqid=0 2024-11-18T23:16:21,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742064_1240 (size=14741) 2024-11-18T23:16:21,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/5c4afd79f3a14b678e14497e0bd6d24a 2024-11-18T23:16:21,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971841858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971841859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/be898f4c67604d56a45a7c7a43b22827 is 50, key is test_row_0/B:col10/1731971781439/Put/seqid=0 2024-11-18T23:16:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971841869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971841873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971841873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742065_1241 (size=12301) 2024-11-18T23:16:21,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/be898f4c67604d56a45a7c7a43b22827 2024-11-18T23:16:21,938 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/f767bde680d246ff94ee0d9484092976 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f767bde680d246ff94ee0d9484092976 2024-11-18T23:16:21,973 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into f767bde680d246ff94ee0d9484092976(size=12.3 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:21,973 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:21,973 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971781300; duration=0sec 2024-11-18T23:16:21,973 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:21,973 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:21,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ae24754686f44c309c2fe3c0ab464d6d is 50, key is test_row_0/C:col10/1731971781439/Put/seqid=0 2024-11-18T23:16:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971841980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:21,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971841984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971841990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971841990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971841992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,003 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742066_1242 (size=12301) 2024-11-18T23:16:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,014 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=634 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ae24754686f44c309c2fe3c0ab464d6d 2024-11-18T23:16:22,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/5c4afd79f3a14b678e14497e0bd6d24a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/5c4afd79f3a14b678e14497e0bd6d24a 2024-11-18T23:16:22,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/5c4afd79f3a14b678e14497e0bd6d24a, entries=200, sequenceid=634, filesize=14.4 K 2024-11-18T23:16:22,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/be898f4c67604d56a45a7c7a43b22827 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/be898f4c67604d56a45a7c7a43b22827 2024-11-18T23:16:22,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/be898f4c67604d56a45a7c7a43b22827, entries=150, sequenceid=634, filesize=12.0 K 2024-11-18T23:16:22,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/ae24754686f44c309c2fe3c0ab464d6d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ae24754686f44c309c2fe3c0ab464d6d 2024-11-18T23:16:22,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ae24754686f44c309c2fe3c0ab464d6d, entries=150, sequenceid=634, filesize=12.0 K 2024-11-18T23:16:22,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d0aae39639c33cd55093c78ecbe9c4f6 in 262ms, sequenceid=634, compaction requested=true 2024-11-18T23:16:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:22,071 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:22,071 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:22,072 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:22,073 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:22,073 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:22,073 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f767bde680d246ff94ee0d9484092976, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f750e4da358e4df3a785deb1f6c928b8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/be898f4c67604d56a45a7c7a43b22827] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.4 K 2024-11-18T23:16:22,073 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:22,073 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:22,073 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:22,073 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/63de77569f0440bebd9994871035adfd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/33cb6e71393b4388af2f7b5dc08ce500, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/5c4afd79f3a14b678e14497e0bd6d24a] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=38.8 K 2024-11-18T23:16:22,074 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f767bde680d246ff94ee0d9484092976, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731971780128 2024-11-18T23:16:22,074 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 63de77569f0440bebd9994871035adfd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731971780128 2024-11-18T23:16:22,074 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f750e4da358e4df3a785deb1f6c928b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1731971780216 2024-11-18T23:16:22,074 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 33cb6e71393b4388af2f7b5dc08ce500, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1731971780216 2024-11-18T23:16:22,074 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting be898f4c67604d56a45a7c7a43b22827, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731971781439 2024-11-18T23:16:22,075 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c4afd79f3a14b678e14497e0bd6d24a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731971781439 2024-11-18T23:16:22,084 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:22,086 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/8313cf15935e4df39d6cb1f7f9dcc93e is 50, key is test_row_0/B:col10/1731971781439/Put/seqid=0 2024-11-18T23:16:22,091 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:22,091 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/d3d7c2841ee64b23ab2026f9f4e173d2 is 50, key is test_row_0/A:col10/1731971781439/Put/seqid=0 2024-11-18T23:16:22,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742067_1243 (size=12745) 2024-11-18T23:16:22,141 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/8313cf15935e4df39d6cb1f7f9dcc93e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/8313cf15935e4df39d6cb1f7f9dcc93e 2024-11-18T23:16:22,148 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into 8313cf15935e4df39d6cb1f7f9dcc93e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:22,148 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:22,148 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971782071; duration=0sec 2024-11-18T23:16:22,149 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:22,149 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:22,149 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:22,150 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:22,150 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742068_1244 (size=12745) 2024-11-18T23:16:22,160 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:22,161 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ebaea5f1f7034671bdd03f1923765b1f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/e290962ae96d41f785879936d7195d81, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ae24754686f44c309c2fe3c0ab464d6d] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.4 K 2024-11-18T23:16:22,163 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebaea5f1f7034671bdd03f1923765b1f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731971780128 2024-11-18T23:16:22,163 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/d3d7c2841ee64b23ab2026f9f4e173d2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d3d7c2841ee64b23ab2026f9f4e173d2 2024-11-18T23:16:22,163 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e290962ae96d41f785879936d7195d81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=617, earliestPutTs=1731971780216 2024-11-18T23:16:22,163 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae24754686f44c309c2fe3c0ab464d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731971781439 2024-11-18T23:16:22,171 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into d3d7c2841ee64b23ab2026f9f4e173d2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:22,171 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:22,171 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971782071; duration=0sec 2024-11-18T23:16:22,171 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:22,171 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:22,214 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#205 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:22,214 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/1ccf0b6c950f435daa01dd9518f873ad is 50, key is test_row_0/C:col10/1731971781439/Put/seqid=0 2024-11-18T23:16:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:22,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T23:16:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:22,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:22,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/4f512133578944708c7c9ca798371c31 is 50, key is test_row_0/A:col10/1731971781863/Put/seqid=0 2024-11-18T23:16:22,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742069_1245 (size=12745) 2024-11-18T23:16:22,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971842248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971842251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971842252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,263 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/1ccf0b6c950f435daa01dd9518f873ad as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/1ccf0b6c950f435daa01dd9518f873ad 2024-11-18T23:16:22,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971842257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,264 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971842261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,276 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into 1ccf0b6c950f435daa01dd9518f873ad(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:22,276 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:22,276 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971782071; duration=0sec 2024-11-18T23:16:22,277 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:22,277 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:22,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742070_1246 (size=14741) 2024-11-18T23:16:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971842367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971842367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971842368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971842368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971842378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971842577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971842578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971842578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971842578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971842587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/4f512133578944708c7c9ca798371c31 2024-11-18T23:16:22,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/4a0f9a21f58d4099b347fae53a7fda4a is 50, key is test_row_0/B:col10/1731971781863/Put/seqid=0 2024-11-18T23:16:22,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742071_1247 (size=12301) 2024-11-18T23:16:22,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/4a0f9a21f58d4099b347fae53a7fda4a 2024-11-18T23:16:22,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9e500e7f4a8647b3b1868d785db42670 is 50, key is test_row_0/C:col10/1731971781863/Put/seqid=0 2024-11-18T23:16:22,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742072_1248 (size=12301) 2024-11-18T23:16:22,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=655 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9e500e7f4a8647b3b1868d785db42670 2024-11-18T23:16:22,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/4f512133578944708c7c9ca798371c31 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4f512133578944708c7c9ca798371c31 2024-11-18T23:16:22,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4f512133578944708c7c9ca798371c31, entries=200, sequenceid=655, filesize=14.4 K 2024-11-18T23:16:22,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/4a0f9a21f58d4099b347fae53a7fda4a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/4a0f9a21f58d4099b347fae53a7fda4a 2024-11-18T23:16:22,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/4a0f9a21f58d4099b347fae53a7fda4a, entries=150, sequenceid=655, filesize=12.0 K 2024-11-18T23:16:22,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9e500e7f4a8647b3b1868d785db42670 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9e500e7f4a8647b3b1868d785db42670 2024-11-18T23:16:22,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9e500e7f4a8647b3b1868d785db42670, entries=150, sequenceid=655, filesize=12.0 K 2024-11-18T23:16:22,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d0aae39639c33cd55093c78ecbe9c4f6 in 614ms, sequenceid=655, compaction requested=false 2024-11-18T23:16:22,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:22,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:16:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:22,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:22,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/14e1ef6474e54411951286099063d259 is 50, key is test_row_0/A:col10/1731971782255/Put/seqid=0 2024-11-18T23:16:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971842917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,932 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T23:16:22,932 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-18T23:16:22,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971842920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971842924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971842924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971842928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:22,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742073_1249 (size=12301) 2024-11-18T23:16:22,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/14e1ef6474e54411951286099063d259 2024-11-18T23:16:22,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/64dcf323e39b45a7aa060b7526df85c1 is 50, key is test_row_0/B:col10/1731971782255/Put/seqid=0 2024-11-18T23:16:23,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742074_1250 (size=12301) 2024-11-18T23:16:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971843038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971843037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971843048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971843049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971843052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971843264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971843264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971843269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971843272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,280 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971843281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-18T23:16:23,396 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:23,400 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:23,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-18T23:16:23,406 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:23,407 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:23,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:23,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T23:16:23,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/64dcf323e39b45a7aa060b7526df85c1 2024-11-18T23:16:23,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9470dac1c2824e0098ec3c754fdc85bd is 50, key is test_row_0/C:col10/1731971782255/Put/seqid=0 2024-11-18T23:16:23,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T23:16:23,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742075_1251 (size=12301) 2024-11-18T23:16:23,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T23:16:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:23,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:23,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=672 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9470dac1c2824e0098ec3c754fdc85bd 2024-11-18T23:16:23,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:23,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971843588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971843592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971843592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971843592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/14e1ef6474e54411951286099063d259 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/14e1ef6474e54411951286099063d259 2024-11-18T23:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971843605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:23,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:23,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:23,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/14e1ef6474e54411951286099063d259, entries=150, sequenceid=672, filesize=12.0 K 2024-11-18T23:16:23,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/64dcf323e39b45a7aa060b7526df85c1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/64dcf323e39b45a7aa060b7526df85c1 2024-11-18T23:16:23,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/64dcf323e39b45a7aa060b7526df85c1, entries=150, sequenceid=672, filesize=12.0 K 2024-11-18T23:16:23,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9470dac1c2824e0098ec3c754fdc85bd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9470dac1c2824e0098ec3c754fdc85bd 2024-11-18T23:16:23,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9470dac1c2824e0098ec3c754fdc85bd, entries=150, sequenceid=672, filesize=12.0 K 2024-11-18T23:16:23,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d0aae39639c33cd55093c78ecbe9c4f6 in 803ms, sequenceid=672, compaction requested=true 2024-11-18T23:16:23,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:23,695 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:23,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:23,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:23,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:23,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:23,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:23,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:23,697 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:23,708 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:23,708 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:23,708 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:23,709 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d3d7c2841ee64b23ab2026f9f4e173d2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4f512133578944708c7c9ca798371c31, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/14e1ef6474e54411951286099063d259] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=38.9 K 2024-11-18T23:16:23,710 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:23,711 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:23,711 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:23,711 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/8313cf15935e4df39d6cb1f7f9dcc93e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/4a0f9a21f58d4099b347fae53a7fda4a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/64dcf323e39b45a7aa060b7526df85c1] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.5 K 2024-11-18T23:16:23,719 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 8313cf15935e4df39d6cb1f7f9dcc93e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731971781439 2024-11-18T23:16:23,720 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d7c2841ee64b23ab2026f9f4e173d2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731971781439 2024-11-18T23:16:23,720 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a0f9a21f58d4099b347fae53a7fda4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1731971781859 2024-11-18T23:16:23,725 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 64dcf323e39b45a7aa060b7526df85c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731971782252 2024-11-18T23:16:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T23:16:23,733 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f512133578944708c7c9ca798371c31, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1731971781859 2024-11-18T23:16:23,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-18T23:16:23,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:23,740 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e1ef6474e54411951286099063d259, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731971782252 2024-11-18T23:16:23,740 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:16:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:23,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:23,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/159ad81677eb420f9933f71e56a56b8a is 50, key is test_row_0/A:col10/1731971782922/Put/seqid=0 2024-11-18T23:16:23,813 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:23,813 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/14069a51a27e44928641c0b103dc05b1 is 50, key is test_row_0/B:col10/1731971782255/Put/seqid=0 2024-11-18T23:16:23,818 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#214 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:23,819 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/77c4eae1ff2649db9652a58b8d296845 is 50, key is test_row_0/A:col10/1731971782255/Put/seqid=0 2024-11-18T23:16:23,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742076_1252 (size=12301) 2024-11-18T23:16:23,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742078_1254 (size=12847) 2024-11-18T23:16:23,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742077_1253 (size=12847) 2024-11-18T23:16:24,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T23:16:24,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:24,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971844147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971844149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971844149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971844148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971844153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,232 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/159ad81677eb420f9933f71e56a56b8a 2024-11-18T23:16:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971844267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971844267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971844268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971844268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971844268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,284 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/77c4eae1ff2649db9652a58b8d296845 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77c4eae1ff2649db9652a58b8d296845 2024-11-18T23:16:24,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/734491cbbd7343eeb07c8ae313d8df4b is 50, key is test_row_0/B:col10/1731971782922/Put/seqid=0 2024-11-18T23:16:24,294 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/14069a51a27e44928641c0b103dc05b1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/14069a51a27e44928641c0b103dc05b1 2024-11-18T23:16:24,300 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into 77c4eae1ff2649db9652a58b8d296845(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:24,300 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:24,300 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971783695; duration=0sec 2024-11-18T23:16:24,300 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:24,300 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:24,301 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:24,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37347 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:24,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:24,305 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:24,306 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/1ccf0b6c950f435daa01dd9518f873ad, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9e500e7f4a8647b3b1868d785db42670, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9470dac1c2824e0098ec3c754fdc85bd] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.5 K 2024-11-18T23:16:24,307 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ccf0b6c950f435daa01dd9518f873ad, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=634, earliestPutTs=1731971781439 2024-11-18T23:16:24,307 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e500e7f4a8647b3b1868d785db42670, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=655, earliestPutTs=1731971781859 2024-11-18T23:16:24,307 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9470dac1c2824e0098ec3c754fdc85bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731971782252 2024-11-18T23:16:24,309 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into 14069a51a27e44928641c0b103dc05b1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:24,309 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:24,309 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971783695; duration=0sec 2024-11-18T23:16:24,309 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:24,309 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:24,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742079_1255 (size=12301) 2024-11-18T23:16:24,311 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/734491cbbd7343eeb07c8ae313d8df4b 2024-11-18T23:16:24,329 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:24,330 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a4b0320c428740f3ad7e27ba16605bdd is 50, key is test_row_0/C:col10/1731971782255/Put/seqid=0 2024-11-18T23:16:24,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b8f07cdc0d464208b608784de8e15b76 is 50, key is test_row_0/C:col10/1731971782922/Put/seqid=0 2024-11-18T23:16:24,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742080_1256 (size=12847) 2024-11-18T23:16:24,420 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a4b0320c428740f3ad7e27ba16605bdd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a4b0320c428740f3ad7e27ba16605bdd 2024-11-18T23:16:24,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742081_1257 (size=12301) 2024-11-18T23:16:24,443 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into a4b0320c428740f3ad7e27ba16605bdd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:24,443 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:24,443 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971783695; duration=0sec 2024-11-18T23:16:24,443 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:24,443 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:24,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971844478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971844478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971844478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971844478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971844478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T23:16:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971844791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971844791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971844791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971844792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971844799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:24,847 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=688 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b8f07cdc0d464208b608784de8e15b76 2024-11-18T23:16:24,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/159ad81677eb420f9933f71e56a56b8a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/159ad81677eb420f9933f71e56a56b8a 2024-11-18T23:16:24,899 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/159ad81677eb420f9933f71e56a56b8a, entries=150, sequenceid=688, filesize=12.0 K 2024-11-18T23:16:24,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/734491cbbd7343eeb07c8ae313d8df4b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/734491cbbd7343eeb07c8ae313d8df4b 2024-11-18T23:16:24,906 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/734491cbbd7343eeb07c8ae313d8df4b, entries=150, sequenceid=688, filesize=12.0 K 2024-11-18T23:16:24,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b8f07cdc0d464208b608784de8e15b76 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8f07cdc0d464208b608784de8e15b76 2024-11-18T23:16:24,916 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8f07cdc0d464208b608784de8e15b76, entries=150, sequenceid=688, filesize=12.0 K 2024-11-18T23:16:24,924 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d0aae39639c33cd55093c78ecbe9c4f6 in 1184ms, sequenceid=688, compaction requested=false 2024-11-18T23:16:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:24,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-18T23:16:24,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-18T23:16:24,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-18T23:16:24,965 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5330 sec 2024-11-18T23:16:24,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5760 sec 2024-11-18T23:16:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:25,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T23:16:25,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:25,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:25,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:25,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:25,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:25,313 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:25,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/2a8ce31c803d4e659cda3343a5fb9d25 is 50, key is test_row_0/A:col10/1731971785304/Put/seqid=0 2024-11-18T23:16:25,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742082_1258 (size=12301) 2024-11-18T23:16:25,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=709 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/2a8ce31c803d4e659cda3343a5fb9d25 2024-11-18T23:16:25,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971845363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971845363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,376 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971845368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,377 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971845374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971845376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/dda5bd96336741c08aa5c0afd862a5d5 is 50, key is test_row_0/B:col10/1731971785304/Put/seqid=0 2024-11-18T23:16:25,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742083_1259 (size=12301) 2024-11-18T23:16:25,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=709 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/dda5bd96336741c08aa5c0afd862a5d5 2024-11-18T23:16:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971845487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971845487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971845488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971845489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971845497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a0c54a78d1a140808d76defea9e9de76 is 50, key is test_row_0/C:col10/1731971785304/Put/seqid=0 2024-11-18T23:16:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-18T23:16:25,577 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:25,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:25,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742084_1260 (size=12301) 2024-11-18T23:16:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-18T23:16:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T23:16:25,596 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:25,603 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:25,604 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T23:16:25,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971845698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971845699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971845700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971845704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971845718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:25,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T23:16:25,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:25,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:25,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:25,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:25,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T23:16:25,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T23:16:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:25,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:25,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:25,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=709 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a0c54a78d1a140808d76defea9e9de76 2024-11-18T23:16:26,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/2a8ce31c803d4e659cda3343a5fb9d25 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/2a8ce31c803d4e659cda3343a5fb9d25 2024-11-18T23:16:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971846007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971846007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/2a8ce31c803d4e659cda3343a5fb9d25, entries=150, sequenceid=709, filesize=12.0 K 2024-11-18T23:16:26,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971846017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/dda5bd96336741c08aa5c0afd862a5d5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dda5bd96336741c08aa5c0afd862a5d5 2024-11-18T23:16:26,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971846017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971846040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dda5bd96336741c08aa5c0afd862a5d5, entries=150, sequenceid=709, filesize=12.0 K 2024-11-18T23:16:26,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a0c54a78d1a140808d76defea9e9de76 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a0c54a78d1a140808d76defea9e9de76 2024-11-18T23:16:26,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a0c54a78d1a140808d76defea9e9de76, entries=150, sequenceid=709, filesize=12.0 K 2024-11-18T23:16:26,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for d0aae39639c33cd55093c78ecbe9c4f6 in 764ms, sequenceid=709, compaction requested=true 2024-11-18T23:16:26,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:26,074 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:26,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:26,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:26,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:26,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:26,075 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:26,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:26,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:26,081 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:26,081 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:26,082 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:26,082 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77c4eae1ff2649db9652a58b8d296845, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/159ad81677eb420f9933f71e56a56b8a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/2a8ce31c803d4e659cda3343a5fb9d25] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.6 K 2024-11-18T23:16:26,084 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c4eae1ff2649db9652a58b8d296845, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731971782252 2024-11-18T23:16:26,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-18T23:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:26,088 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T23:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:26,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:26,089 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 159ad81677eb420f9933f71e56a56b8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1731971782918 2024-11-18T23:16:26,090 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a8ce31c803d4e659cda3343a5fb9d25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=709, earliestPutTs=1731971784141 2024-11-18T23:16:26,091 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:26,091 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:26,091 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:26,091 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/14069a51a27e44928641c0b103dc05b1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/734491cbbd7343eeb07c8ae313d8df4b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dda5bd96336741c08aa5c0afd862a5d5] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.6 K 2024-11-18T23:16:26,097 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14069a51a27e44928641c0b103dc05b1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731971782252 2024-11-18T23:16:26,104 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 734491cbbd7343eeb07c8ae313d8df4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1731971782918 2024-11-18T23:16:26,104 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting dda5bd96336741c08aa5c0afd862a5d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=709, earliestPutTs=1731971784141 2024-11-18T23:16:26,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/274c976163204170be23c0110ba3d061 is 50, key is test_row_0/A:col10/1731971785372/Put/seqid=0 2024-11-18T23:16:26,122 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:26,123 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/d9c9e7c2eb8240169a1951ae9580ee08 is 50, key is test_row_0/A:col10/1731971785304/Put/seqid=0 2024-11-18T23:16:26,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742086_1262 (size=12949) 2024-11-18T23:16:26,182 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:26,183 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/e67b30829fc7442394a4454d6fe2b6e3 is 50, key is test_row_0/B:col10/1731971785304/Put/seqid=0 2024-11-18T23:16:26,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742085_1261 (size=12301) 2024-11-18T23:16:26,191 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/274c976163204170be23c0110ba3d061 2024-11-18T23:16:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T23:16:26,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742087_1263 (size=12949) 2024-11-18T23:16:26,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/9bf3430de81943878fdb230a00fe756e is 50, key is test_row_0/B:col10/1731971785372/Put/seqid=0 2024-11-18T23:16:26,252 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/e67b30829fc7442394a4454d6fe2b6e3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e67b30829fc7442394a4454d6fe2b6e3 2024-11-18T23:16:26,264 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into e67b30829fc7442394a4454d6fe2b6e3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:26,264 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:26,264 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971786074; duration=0sec 2024-11-18T23:16:26,264 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:26,264 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:26,264 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:26,266 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:26,266 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:26,266 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:26,266 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a4b0320c428740f3ad7e27ba16605bdd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8f07cdc0d464208b608784de8e15b76, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a0c54a78d1a140808d76defea9e9de76] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.6 K 2024-11-18T23:16:26,266 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4b0320c428740f3ad7e27ba16605bdd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=672, earliestPutTs=1731971782252 2024-11-18T23:16:26,267 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8f07cdc0d464208b608784de8e15b76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=688, earliestPutTs=1731971782918 2024-11-18T23:16:26,267 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0c54a78d1a140808d76defea9e9de76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=709, earliestPutTs=1731971784141 2024-11-18T23:16:26,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742088_1264 (size=12301) 2024-11-18T23:16:26,284 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/9bf3430de81943878fdb230a00fe756e 2024-11-18T23:16:26,284 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#225 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:26,289 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/7dd66fd9f45c48eab44e4b8ea5d3c64e is 50, key is test_row_0/C:col10/1731971785304/Put/seqid=0 2024-11-18T23:16:26,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742089_1265 (size=12949) 2024-11-18T23:16:26,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/c67e9c21cf8f494caf4c1c32543e1b84 is 50, key is test_row_0/C:col10/1731971785372/Put/seqid=0 2024-11-18T23:16:26,318 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/7dd66fd9f45c48eab44e4b8ea5d3c64e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/7dd66fd9f45c48eab44e4b8ea5d3c64e 2024-11-18T23:16:26,326 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into 7dd66fd9f45c48eab44e4b8ea5d3c64e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:26,326 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:26,326 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971786074; duration=0sec 2024-11-18T23:16:26,326 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:26,326 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:26,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742090_1266 (size=12301) 2024-11-18T23:16:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:26,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971846568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971846569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971846571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971846569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,582 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/d9c9e7c2eb8240169a1951ae9580ee08 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d9c9e7c2eb8240169a1951ae9580ee08 2024-11-18T23:16:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971846578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,600 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into d9c9e7c2eb8240169a1951ae9580ee08(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:26,601 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:26,601 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971786073; duration=0sec 2024-11-18T23:16:26,601 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:26,601 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:26,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971846689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971846689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,691 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971846689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971846689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971846690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T23:16:26,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/c67e9c21cf8f494caf4c1c32543e1b84 2024-11-18T23:16:26,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/274c976163204170be23c0110ba3d061 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/274c976163204170be23c0110ba3d061 2024-11-18T23:16:26,833 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/274c976163204170be23c0110ba3d061, entries=150, sequenceid=724, filesize=12.0 K 2024-11-18T23:16:26,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/9bf3430de81943878fdb230a00fe756e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/9bf3430de81943878fdb230a00fe756e 2024-11-18T23:16:26,879 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/9bf3430de81943878fdb230a00fe756e, entries=150, sequenceid=724, filesize=12.0 K 2024-11-18T23:16:26,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/c67e9c21cf8f494caf4c1c32543e1b84 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c67e9c21cf8f494caf4c1c32543e1b84 2024-11-18T23:16:26,902 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c67e9c21cf8f494caf4c1c32543e1b84, entries=150, sequenceid=724, filesize=12.0 K 2024-11-18T23:16:26,903 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for d0aae39639c33cd55093c78ecbe9c4f6 in 815ms, sequenceid=724, compaction requested=false 2024-11-18T23:16:26,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:26,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:26,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-18T23:16:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-18T23:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:26,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T23:16:26,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-18T23:16:26,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3080 sec 2024-11-18T23:16:26,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:26,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:26,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:26,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:26,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:26,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:26,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/3b7642bc292d4e5396519d5445db46a1 is 50, key is test_row_0/A:col10/1731971786913/Put/seqid=0 2024-11-18T23:16:26,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.3500 sec 2024-11-18T23:16:26,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971846939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971846939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971846946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742091_1267 (size=12301) 2024-11-18T23:16:26,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/3b7642bc292d4e5396519d5445db46a1 2024-11-18T23:16:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971846947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971846948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:26,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/83d71311cc4949ca84c90659c12ca368 is 50, key is test_row_0/B:col10/1731971786913/Put/seqid=0 2024-11-18T23:16:26,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742092_1268 (size=12301) 2024-11-18T23:16:26,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/83d71311cc4949ca84c90659c12ca368 2024-11-18T23:16:26,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/d671872c6f944b2b80475b8cc7e91648 is 50, key is test_row_0/C:col10/1731971786913/Put/seqid=0 2024-11-18T23:16:27,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742093_1269 (size=12301) 2024-11-18T23:16:27,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=748 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/d671872c6f944b2b80475b8cc7e91648 2024-11-18T23:16:27,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971847057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971847057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971847057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971847058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971847060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,065 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/3b7642bc292d4e5396519d5445db46a1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/3b7642bc292d4e5396519d5445db46a1 2024-11-18T23:16:27,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/3b7642bc292d4e5396519d5445db46a1, entries=150, sequenceid=748, filesize=12.0 K 2024-11-18T23:16:27,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/83d71311cc4949ca84c90659c12ca368 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/83d71311cc4949ca84c90659c12ca368 2024-11-18T23:16:27,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/83d71311cc4949ca84c90659c12ca368, entries=150, sequenceid=748, filesize=12.0 K 2024-11-18T23:16:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/d671872c6f944b2b80475b8cc7e91648 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d671872c6f944b2b80475b8cc7e91648 2024-11-18T23:16:27,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d671872c6f944b2b80475b8cc7e91648, entries=150, sequenceid=748, filesize=12.0 K 2024-11-18T23:16:27,149 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for d0aae39639c33cd55093c78ecbe9c4f6 in 231ms, sequenceid=748, compaction requested=true 2024-11-18T23:16:27,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:27,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:27,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:27,149 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:27,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:27,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:27,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:27,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:27,151 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:27,157 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:27,157 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:27,157 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:27,157 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d9c9e7c2eb8240169a1951ae9580ee08, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/274c976163204170be23c0110ba3d061, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/3b7642bc292d4e5396519d5445db46a1] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.7 K 2024-11-18T23:16:27,161 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9c9e7c2eb8240169a1951ae9580ee08, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=709, earliestPutTs=1731971784141 2024-11-18T23:16:27,161 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:27,161 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:27,161 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:27,161 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e67b30829fc7442394a4454d6fe2b6e3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/9bf3430de81943878fdb230a00fe756e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/83d71311cc4949ca84c90659c12ca368] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.7 K 2024-11-18T23:16:27,162 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 274c976163204170be23c0110ba3d061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731971785362 2024-11-18T23:16:27,162 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e67b30829fc7442394a4454d6fe2b6e3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=709, earliestPutTs=1731971784141 2024-11-18T23:16:27,163 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b7642bc292d4e5396519d5445db46a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=748, earliestPutTs=1731971786568 2024-11-18T23:16:27,163 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf3430de81943878fdb230a00fe756e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731971785362 2024-11-18T23:16:27,165 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d71311cc4949ca84c90659c12ca368, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=748, earliestPutTs=1731971786568 2024-11-18T23:16:27,190 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:27,191 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/4617b4b20ff6426a92683677fe4c8d24 is 50, key is test_row_0/A:col10/1731971786913/Put/seqid=0 2024-11-18T23:16:27,198 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:27,199 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/04022dceea364b1290c36093efcb5912 is 50, key is test_row_0/B:col10/1731971786913/Put/seqid=0 2024-11-18T23:16:27,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742094_1270 (size=13051) 2024-11-18T23:16:27,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742095_1271 (size=13051) 2024-11-18T23:16:27,267 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/04022dceea364b1290c36093efcb5912 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/04022dceea364b1290c36093efcb5912 2024-11-18T23:16:27,284 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into 04022dceea364b1290c36093efcb5912(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:27,284 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:27,284 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971787149; duration=0sec 2024-11-18T23:16:27,285 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:27,285 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:27,285 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:27,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T23:16:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:27,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:27,294 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:27,294 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:27,295 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:27,295 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/7dd66fd9f45c48eab44e4b8ea5d3c64e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c67e9c21cf8f494caf4c1c32543e1b84, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d671872c6f944b2b80475b8cc7e91648] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.7 K 2024-11-18T23:16:27,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/8c73240a4f4544a3a1b59f6540467af2 is 50, key is test_row_0/A:col10/1731971787276/Put/seqid=0 2024-11-18T23:16:27,299 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dd66fd9f45c48eab44e4b8ea5d3c64e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=709, earliestPutTs=1731971784141 2024-11-18T23:16:27,300 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c67e9c21cf8f494caf4c1c32543e1b84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=724, earliestPutTs=1731971785362 2024-11-18T23:16:27,305 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d671872c6f944b2b80475b8cc7e91648, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=748, earliestPutTs=1731971786568 2024-11-18T23:16:27,338 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#233 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:27,339 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b8b8e7eb072945a0a2f703da038f1aa5 is 50, key is test_row_0/C:col10/1731971786913/Put/seqid=0 2024-11-18T23:16:27,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742096_1272 (size=12301) 2024-11-18T23:16:27,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/8c73240a4f4544a3a1b59f6540467af2 2024-11-18T23:16:27,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971847358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971847366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,378 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971847372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971847376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971847384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742097_1273 (size=13051) 2024-11-18T23:16:27,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/c074b6fe2248496289d0b90f9a606598 is 50, key is test_row_0/B:col10/1731971787276/Put/seqid=0 2024-11-18T23:16:27,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742098_1274 (size=12301) 2024-11-18T23:16:27,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971847488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971847488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971847488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971847497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971847500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,504 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,683 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/4617b4b20ff6426a92683677fe4c8d24 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4617b4b20ff6426a92683677fe4c8d24 2024-11-18T23:16:27,690 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into 4617b4b20ff6426a92683677fe4c8d24(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:27,690 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:27,690 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971787149; duration=0sec 2024-11-18T23:16:27,690 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:27,690 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:27,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971847698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971847698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971847704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971847708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971847711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:27,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:27,720 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-18T23:16:27,750 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:27,751 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-18T23:16:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:27,753 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:27,760 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:27,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:27,821 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b8b8e7eb072945a0a2f703da038f1aa5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8b8e7eb072945a0a2f703da038f1aa5 2024-11-18T23:16:27,845 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into b8b8e7eb072945a0a2f703da038f1aa5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:27,845 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:27,845 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971787149; duration=0sec 2024-11-18T23:16:27,845 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:27,845 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:27,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/c074b6fe2248496289d0b90f9a606598 2024-11-18T23:16:27,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/2fb2e53addaf4e14a9df4412de5660c0 is 50, key is test_row_0/C:col10/1731971787276/Put/seqid=0 2024-11-18T23:16:27,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:27,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:27,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742099_1275 (size=12301) 2024-11-18T23:16:28,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971848009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971848011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971848024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971848024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971848032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:28,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:28,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:28,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:28,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:28,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/2fb2e53addaf4e14a9df4412de5660c0 2024-11-18T23:16:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:28,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/8c73240a4f4544a3a1b59f6540467af2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/8c73240a4f4544a3a1b59f6540467af2 2024-11-18T23:16:28,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/8c73240a4f4544a3a1b59f6540467af2, entries=150, sequenceid=763, filesize=12.0 K 2024-11-18T23:16:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/c074b6fe2248496289d0b90f9a606598 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c074b6fe2248496289d0b90f9a606598 2024-11-18T23:16:28,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:28,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:28,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c074b6fe2248496289d0b90f9a606598, entries=150, sequenceid=763, filesize=12.0 K 2024-11-18T23:16:28,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/2fb2e53addaf4e14a9df4412de5660c0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/2fb2e53addaf4e14a9df4412de5660c0 2024-11-18T23:16:28,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/2fb2e53addaf4e14a9df4412de5660c0, entries=150, sequenceid=763, filesize=12.0 K 2024-11-18T23:16:28,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for d0aae39639c33cd55093c78ecbe9c4f6 in 1167ms, sequenceid=763, compaction requested=false 2024-11-18T23:16:28,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:28,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-18T23:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:28,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/f84154e71f634daa852f5c141c472727 is 50, key is test_row_0/A:col10/1731971788533/Put/seqid=0 2024-11-18T23:16:28,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:28,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971848554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971848550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971848563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971848564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971848569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742100_1276 (size=14741) 2024-11-18T23:16:28,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971848683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971848689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971848690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971848690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971848692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:28,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:28,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:28,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:28,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:28,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:28,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:28,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971848909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971848909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971848911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971848911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971848911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:28,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=787 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/f84154e71f634daa852f5c141c472727 2024-11-18T23:16:29,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/aed0ffab56404a0eb2d211214bf59c4b is 50, key is test_row_0/B:col10/1731971788533/Put/seqid=0 2024-11-18T23:16:29,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742101_1277 (size=12301) 2024-11-18T23:16:29,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:29,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:29,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971849217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971849227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971849227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971849227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971849227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=787 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/aed0ffab56404a0eb2d211214bf59c4b 2024-11-18T23:16:29,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:29,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:29,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a9b68a972d884d39970004c798d09c5b is 50, key is test_row_0/C:col10/1731971788533/Put/seqid=0 2024-11-18T23:16:29,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742102_1278 (size=12301) 2024-11-18T23:16:29,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=787 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a9b68a972d884d39970004c798d09c5b 2024-11-18T23:16:29,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/f84154e71f634daa852f5c141c472727 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f84154e71f634daa852f5c141c472727 2024-11-18T23:16:29,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f84154e71f634daa852f5c141c472727, entries=200, sequenceid=787, filesize=14.4 K 2024-11-18T23:16:29,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/aed0ffab56404a0eb2d211214bf59c4b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/aed0ffab56404a0eb2d211214bf59c4b 2024-11-18T23:16:29,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/aed0ffab56404a0eb2d211214bf59c4b, entries=150, sequenceid=787, filesize=12.0 K 2024-11-18T23:16:29,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/a9b68a972d884d39970004c798d09c5b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a9b68a972d884d39970004c798d09c5b 2024-11-18T23:16:29,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a9b68a972d884d39970004c798d09c5b, entries=150, sequenceid=787, filesize=12.0 K 2024-11-18T23:16:29,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for d0aae39639c33cd55093c78ecbe9c4f6 in 1161ms, sequenceid=787, compaction requested=true 2024-11-18T23:16:29,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:29,696 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:29,696 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:29,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:29,701 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:29,701 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:29,701 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,701 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4617b4b20ff6426a92683677fe4c8d24, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/8c73240a4f4544a3a1b59f6540467af2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f84154e71f634daa852f5c141c472727] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=39.2 K 2024-11-18T23:16:29,703 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:29,703 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:29,703 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,703 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/04022dceea364b1290c36093efcb5912, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c074b6fe2248496289d0b90f9a606598, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/aed0ffab56404a0eb2d211214bf59c4b] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.8 K 2024-11-18T23:16:29,703 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4617b4b20ff6426a92683677fe4c8d24, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=748, earliestPutTs=1731971786568 2024-11-18T23:16:29,705 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 04022dceea364b1290c36093efcb5912, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=748, earliestPutTs=1731971786568 2024-11-18T23:16:29,705 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c73240a4f4544a3a1b59f6540467af2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1731971787275 2024-11-18T23:16:29,710 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c074b6fe2248496289d0b90f9a606598, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1731971787275 2024-11-18T23:16:29,712 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f84154e71f634daa852f5c141c472727, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=787, earliestPutTs=1731971787358 2024-11-18T23:16:29,713 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting aed0ffab56404a0eb2d211214bf59c4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=787, earliestPutTs=1731971787358 2024-11-18T23:16:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:29,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:16:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:29,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:29,756 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:29,757 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/ac6bb53a93754e46909c067af4a3d805 is 50, key is test_row_0/B:col10/1731971788533/Put/seqid=0 2024-11-18T23:16:29,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/9bd49c7e9d314df7adf104be815e477f is 50, key is test_row_0/A:col10/1731971789733/Put/seqid=0 2024-11-18T23:16:29,779 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:29,779 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/a7d8c84661da4e7c8384e6eb520a8b48 is 50, key is test_row_0/A:col10/1731971788533/Put/seqid=0 2024-11-18T23:16:29,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742103_1279 (size=13153) 2024-11-18T23:16:29,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742104_1280 (size=12301) 2024-11-18T23:16:29,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742105_1281 (size=13153) 2024-11-18T23:16:29,844 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/a7d8c84661da4e7c8384e6eb520a8b48 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/a7d8c84661da4e7c8384e6eb520a8b48 2024-11-18T23:16:29,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:29,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971789799, max timestamp=1731971789826Num uniques 859; 2024-11-18T23:16:29,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:29,880 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into a7d8c84661da4e7c8384e6eb520a8b48(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:29,880 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:29,880 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971789695; duration=0sec 2024-11-18T23:16:29,881 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:29,881 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:29,881 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:29,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971789799, max timestamp=1731971789816Num uniques 859; 2024-11-18T23:16:29,881 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:16:29,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971789799, max timestamp=1731971789816Num uniques 859; 2024-11-18T23:16:29,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:16:29,892 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:29,892 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:29,893 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:29,893 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8b8e7eb072945a0a2f703da038f1aa5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/2fb2e53addaf4e14a9df4412de5660c0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a9b68a972d884d39970004c798d09c5b] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=36.8 K 2024-11-18T23:16:29,893 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8b8e7eb072945a0a2f703da038f1aa5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=748, earliestPutTs=1731971786568 2024-11-18T23:16:29,893 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb2e53addaf4e14a9df4412de5660c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=763, earliestPutTs=1731971787275 2024-11-18T23:16:29,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:29,897 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b68a972d884d39970004c798d09c5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=787, earliestPutTs=1731971787358 2024-11-18T23:16:29,915 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:29,916 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/16fded2395e747a7ad069faf3e790e41 is 50, key is test_row_0/C:col10/1731971788533/Put/seqid=0 2024-11-18T23:16:29,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971849924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971849930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971849932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971849933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:29,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971849945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:29,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742106_1282 (size=13153) 2024-11-18T23:16:30,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:30,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971850048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971850048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971850051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971850051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,062 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971850070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:30,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=798 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/9bd49c7e9d314df7adf104be815e477f 2024-11-18T23:16:30,241 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/ac6bb53a93754e46909c067af4a3d805 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ac6bb53a93754e46909c067af4a3d805 2024-11-18T23:16:30,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/d1dab36e26914e869e94becd8ced14ef is 50, key is test_row_0/B:col10/1731971789733/Put/seqid=0 2024-11-18T23:16:30,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971850273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971850273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971850273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971850268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971850287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,316 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into ac6bb53a93754e46909c067af4a3d805(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:30,316 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:30,316 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971789696; duration=0sec 2024-11-18T23:16:30,316 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:30,317 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:30,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742107_1283 (size=12301) 2024-11-18T23:16:30,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:30,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,428 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/16fded2395e747a7ad069faf3e790e41 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/16fded2395e747a7ad069faf3e790e41 2024-11-18T23:16:30,460 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into 16fded2395e747a7ad069faf3e790e41(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:30,460 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:30,460 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971789696; duration=0sec 2024-11-18T23:16:30,461 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:30,461 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:30,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971850601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971850601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971850601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971850602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,616 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971850628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:30,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:30,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:30,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=798 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/d1dab36e26914e869e94becd8ced14ef 2024-11-18T23:16:30,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/5c71d06ccfd047b1b89e60e211b698ce is 50, key is test_row_0/C:col10/1731971789733/Put/seqid=0 2024-11-18T23:16:30,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:30,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742108_1284 (size=12301) 2024-11-18T23:16:30,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:30,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:30,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:30,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:30,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:31,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:31,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971851117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:31,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971851128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:31,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971851128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971851128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:31,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:31,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:31,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:31,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971851152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:31,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=798 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/5c71d06ccfd047b1b89e60e211b698ce 2024-11-18T23:16:31,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/9bd49c7e9d314df7adf104be815e477f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9bd49c7e9d314df7adf104be815e477f 2024-11-18T23:16:31,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9bd49c7e9d314df7adf104be815e477f, entries=150, sequenceid=798, filesize=12.0 K 2024-11-18T23:16:31,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/d1dab36e26914e869e94becd8ced14ef as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/d1dab36e26914e869e94becd8ced14ef 2024-11-18T23:16:31,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/d1dab36e26914e869e94becd8ced14ef, entries=150, sequenceid=798, filesize=12.0 K 2024-11-18T23:16:31,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/5c71d06ccfd047b1b89e60e211b698ce as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/5c71d06ccfd047b1b89e60e211b698ce 2024-11-18T23:16:31,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:31,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:31,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/5c71d06ccfd047b1b89e60e211b698ce, entries=150, sequenceid=798, filesize=12.0 K 2024-11-18T23:16:31,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:31,314 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for d0aae39639c33cd55093c78ecbe9c4f6 in 1573ms, sequenceid=798, compaction requested=false 2024-11-18T23:16:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:31,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-18T23:16:31,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:31,469 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=126.68 KB heapSize=363.23 KB 2024-11-18T23:16:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:31,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/deb87bceafe64223b3b0f942171d7912 is 50, key is test_row_0/A:col10/1731971789933/Put/seqid=0 2024-11-18T23:16:31,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742109_1285 (size=12301) 2024-11-18T23:16:31,543 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/deb87bceafe64223b3b0f942171d7912 2024-11-18T23:16:31,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/79458eefd4d9483ea1a18587b4eb9d62 is 50, key is test_row_0/B:col10/1731971789933/Put/seqid=0 2024-11-18T23:16:31,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742110_1286 (size=12301) 2024-11-18T23:16:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:32,017 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/79458eefd4d9483ea1a18587b4eb9d62 2024-11-18T23:16:32,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/421350a239244389addb2343578d342a is 50, key is test_row_0/C:col10/1731971789933/Put/seqid=0 2024-11-18T23:16:32,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742111_1287 (size=12301) 2024-11-18T23:16:32,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. as already flushing 2024-11-18T23:16:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:32,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971852193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971852193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971852194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971852196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971852200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,209 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971852321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971852321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971852321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971852321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971852324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,500 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=837 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/421350a239244389addb2343578d342a 2024-11-18T23:16:32,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/deb87bceafe64223b3b0f942171d7912 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/deb87bceafe64223b3b0f942171d7912 2024-11-18T23:16:32,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971852537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971852540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971852540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971852540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971852540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,576 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/deb87bceafe64223b3b0f942171d7912, entries=150, sequenceid=837, filesize=12.0 K 2024-11-18T23:16:32,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/79458eefd4d9483ea1a18587b4eb9d62 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/79458eefd4d9483ea1a18587b4eb9d62 2024-11-18T23:16:32,625 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/79458eefd4d9483ea1a18587b4eb9d62, entries=150, sequenceid=837, filesize=12.0 K 2024-11-18T23:16:32,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/421350a239244389addb2343578d342a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/421350a239244389addb2343578d342a 2024-11-18T23:16:32,640 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/421350a239244389addb2343578d342a, entries=150, sequenceid=837, filesize=12.0 K 2024-11-18T23:16:32,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~126.68 KB/129720, heapSize ~363.19 KB/371904, currentSize=60.38 KB/61830 for d0aae39639c33cd55093c78ecbe9c4f6 in 1173ms, sequenceid=837, compaction requested=true 2024-11-18T23:16:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:32,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-18T23:16:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-18T23:16:32,645 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-18T23:16:32,646 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.8830 sec 2024-11-18T23:16:32,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 4.8950 sec 2024-11-18T23:16:32,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:16:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:32,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:32,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:32,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/77af5f5b5ecc4d2ab26a7be8e355d1b6 is 50, key is test_row_0/A:col10/1731971792185/Put/seqid=0 2024-11-18T23:16:32,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971852901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971852908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971852908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971852910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:32,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971852913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:32,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:32,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742112_1288 (size=12301) 2024-11-18T23:16:33,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971853018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971853027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971853027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971853029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971853027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971853237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971853237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971853237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971853247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971853247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:33,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:33,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=850 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/77af5f5b5ecc4d2ab26a7be8e355d1b6 2024-11-18T23:16:33,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/fede8d7a0198403fa286d7ca322a996b is 50, key is test_row_0/B:col10/1731971792185/Put/seqid=0 2024-11-18T23:16:33,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742113_1289 (size=12301) 2024-11-18T23:16:33,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=850 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/fede8d7a0198403fa286d7ca322a996b 2024-11-18T23:16:33,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9473a981f7854754a1f39f4827d212f5 is 50, key is test_row_0/C:col10/1731971792185/Put/seqid=0 2024-11-18T23:16:33,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742114_1290 (size=12301) 2024-11-18T23:16:33,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=850 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9473a981f7854754a1f39f4827d212f5 2024-11-18T23:16:33,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/77af5f5b5ecc4d2ab26a7be8e355d1b6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77af5f5b5ecc4d2ab26a7be8e355d1b6 2024-11-18T23:16:33,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77af5f5b5ecc4d2ab26a7be8e355d1b6, entries=150, sequenceid=850, filesize=12.0 K 2024-11-18T23:16:33,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/fede8d7a0198403fa286d7ca322a996b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/fede8d7a0198403fa286d7ca322a996b 2024-11-18T23:16:33,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/fede8d7a0198403fa286d7ca322a996b, entries=150, sequenceid=850, filesize=12.0 K 2024-11-18T23:16:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9473a981f7854754a1f39f4827d212f5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9473a981f7854754a1f39f4827d212f5 2024-11-18T23:16:33,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9473a981f7854754a1f39f4827d212f5, entries=150, sequenceid=850, filesize=12.0 K 2024-11-18T23:16:33,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d0aae39639c33cd55093c78ecbe9c4f6 in 669ms, sequenceid=850, compaction requested=true 2024-11-18T23:16:33,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:33,536 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:33,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:33,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:33,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:33,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:33,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:33,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:33,537 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:33,537 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:33,537 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:33,537 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/a7d8c84661da4e7c8384e6eb520a8b48, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9bd49c7e9d314df7adf104be815e477f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/deb87bceafe64223b3b0f942171d7912, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77af5f5b5ecc4d2ab26a7be8e355d1b6] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=48.9 K 2024-11-18T23:16:33,538 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a7d8c84661da4e7c8384e6eb520a8b48, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=787, earliestPutTs=1731971787358 2024-11-18T23:16:33,538 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd49c7e9d314df7adf104be815e477f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=798, earliestPutTs=1731971788556 2024-11-18T23:16:33,538 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting deb87bceafe64223b3b0f942171d7912, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1731971789899 2024-11-18T23:16:33,538 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 77af5f5b5ecc4d2ab26a7be8e355d1b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731971792185 2024-11-18T23:16:33,540 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:33,545 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:33,546 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:33,546 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:33,546 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ac6bb53a93754e46909c067af4a3d805, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/d1dab36e26914e869e94becd8ced14ef, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/79458eefd4d9483ea1a18587b4eb9d62, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/fede8d7a0198403fa286d7ca322a996b] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=48.9 K 2024-11-18T23:16:33,547 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac6bb53a93754e46909c067af4a3d805, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=787, earliestPutTs=1731971787358 2024-11-18T23:16:33,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:33,557 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1dab36e26914e869e94becd8ced14ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=798, earliestPutTs=1731971788556 2024-11-18T23:16:33,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T23:16:33,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971792906, max timestamp=1731971793554Num uniques 859; 2024-11-18T23:16:33,564 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79458eefd4d9483ea1a18587b4eb9d62, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1731971789899 2024-11-18T23:16:33,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:33,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971792906, max timestamp=1731971793554Num uniques 859; 2024-11-18T23:16:33,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:16:33,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971792906, max timestamp=1731971793554Num uniques 859; 2024-11-18T23:16:33,567 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:33,567 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting fede8d7a0198403fa286d7ca322a996b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731971792185 2024-11-18T23:16:33,576 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:33,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:33,576 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/86e807558158421c9429cff1e8e623a4 is 50, key is test_row_0/A:col10/1731971792185/Put/seqid=0 2024-11-18T23:16:33,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:33,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:33,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:33,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:33,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:33,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/1dcde74ceb654de9a5838617a8656ebc is 50, key is test_row_0/A:col10/1731971792906/Put/seqid=0 2024-11-18T23:16:33,647 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:33,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742115_1291 (size=13289) 2024-11-18T23:16:33,647 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/f2c3f20127214d6eb97264ba4c64fd07 is 50, key is test_row_0/B:col10/1731971792185/Put/seqid=0 2024-11-18T23:16:33,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742116_1292 (size=12301) 2024-11-18T23:16:33,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971793642, max timestamp=1731971793658Num uniques 859; 2024-11-18T23:16:33,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:33,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971793642, max timestamp=1731971793658Num uniques 859; 2024-11-18T23:16:33,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:33,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971793642, max timestamp=1731971793658Num uniques 859; 2024-11-18T23:16:33,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:16:33,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/1dcde74ceb654de9a5838617a8656ebc 2024-11-18T23:16:33,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742117_1293 (size=13289) 2024-11-18T23:16:33,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/b355d9b474e34bb3a33d447dbbe4e0d8 is 50, key is test_row_0/B:col10/1731971792906/Put/seqid=0 2024-11-18T23:16:33,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742118_1294 (size=12301) 2024-11-18T23:16:33,713 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/f2c3f20127214d6eb97264ba4c64fd07 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f2c3f20127214d6eb97264ba4c64fd07 2024-11-18T23:16:33,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14550, totalHeapSize=33312, min timestamp=1731971793680, max timestamp=1731971793715Num uniques 1018; 2024-11-18T23:16:33,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=14550, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-18T23:16:33,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14555, totalHeapSize=33312, min timestamp=1731971793680, max timestamp=1731971793715Num uniques 1018; 2024-11-18T23:16:33,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46615, new segment data size=14555, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-18T23:16:33,723 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731971793680, max timestamp=1731971793715Num uniques 1018; 2024-11-18T23:16:33,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-18T23:16:33,728 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into f2c3f20127214d6eb97264ba4c64fd07(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:33,728 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:33,728 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=12, startTime=1731971793537; duration=0sec 2024-11-18T23:16:33,728 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:33,728 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:33,728 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:33,730 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:33,730 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:33,730 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:33,730 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/16fded2395e747a7ad069faf3e790e41, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/5c71d06ccfd047b1b89e60e211b698ce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/421350a239244389addb2343578d342a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9473a981f7854754a1f39f4827d212f5] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=48.9 K 2024-11-18T23:16:33,731 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16fded2395e747a7ad069faf3e790e41, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=787, earliestPutTs=1731971787358 2024-11-18T23:16:33,731 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c71d06ccfd047b1b89e60e211b698ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=798, earliestPutTs=1731971788556 2024-11-18T23:16:33,732 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 421350a239244389addb2343578d342a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=837, earliestPutTs=1731971789899 2024-11-18T23:16:33,733 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9473a981f7854754a1f39f4827d212f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731971792185 2024-11-18T23:16:33,759 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#255 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:33,760 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/665f8bb5920645ea9b90864c3db710e8 is 50, key is test_row_0/C:col10/1731971792185/Put/seqid=0 2024-11-18T23:16:33,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742119_1295 (size=13289) 2024-11-18T23:16:33,768 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/665f8bb5920645ea9b90864c3db710e8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/665f8bb5920645ea9b90864c3db710e8 2024-11-18T23:16:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971793750, max timestamp=1731971793769Num uniques 1177; 2024-11-18T23:16:33,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-18T23:16:33,776 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into 665f8bb5920645ea9b90864c3db710e8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:33,776 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:33,776 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=12, startTime=1731971793537; duration=0sec 2024-11-18T23:16:33,776 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:33,776 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:33,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971793750, max timestamp=1731971793769Num uniques 1177; 2024-11-18T23:16:33,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=10392, suffix heap size=150376,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=227 2024-11-18T23:16:33,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971793750, max timestamp=1731971793769Num uniques 1177; 2024-11-18T23:16:33,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=6870, suffix heap size=150376,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=150 2024-11-18T23:16:33,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731971793797, max timestamp=1731971793810Num uniques 1086; 2024-11-18T23:16:33,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-18T23:16:33,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731971793797, max timestamp=1731971793810Num uniques 1086; 2024-11-18T23:16:33,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T23:16:33,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971793797, max timestamp=1731971793818Num uniques 1009; 2024-11-18T23:16:33,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46204, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:33,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731971793852, max timestamp=1731971793879Num uniques 1095; 2024-11-18T23:16:33,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-18T23:16:33,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971793852, max timestamp=1731971793879Num uniques 1009; 2024-11-18T23:16:33,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971793852, max timestamp=1731971793879Num uniques 1045; 2024-11-18T23:16:33,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=6870, suffix heap size=136648,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=150 2024-11-18T23:16:33,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46224, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:33,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731971793924, max timestamp=1731971793937Num uniques 1104; 2024-11-18T23:16:33,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731971793924, max timestamp=1731971793937Num uniques 1009; 2024-11-18T23:16:33,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=9340, suffix heap size=132904,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=204 2024-11-18T23:16:33,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-18T23:16:33,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971793924, max timestamp=1731971793937Num uniques 1009; 2024-11-18T23:16:33,951 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46210, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-18T23:16:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731971793977, max timestamp=1731971794017Num uniques 1063; 2024-11-18T23:16:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-18T23:16:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731971793977, max timestamp=1731971794017Num uniques 1063; 2024-11-18T23:16:34,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-18T23:16:34,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731971793977, max timestamp=1731971794017Num uniques 1009; 2024-11-18T23:16:34,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=9748, suffix heap size=132904,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=213 2024-11-18T23:16:34,083 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/86e807558158421c9429cff1e8e623a4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/86e807558158421c9429cff1e8e623a4 2024-11-18T23:16:34,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971854104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731971794090, max timestamp=1731971794108Num uniques 1072; 2024-11-18T23:16:34,119 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-18T23:16:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/b355d9b474e34bb3a33d447dbbe4e0d8 2024-11-18T23:16:34,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731971794090, max timestamp=1731971794108Num uniques 1072; 2024-11-18T23:16:34,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-18T23:16:34,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731971794090, max timestamp=1731971794108Num uniques 1072; 2024-11-18T23:16:34,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-18T23:16:34,141 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into 86e807558158421c9429cff1e8e623a4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:34,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:34,141 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=12, startTime=1731971793536; duration=0sec 2024-11-18T23:16:34,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:34,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:34,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/04df726e4e214aa981624f6af6a97cd4 is 50, key is test_row_0/C:col10/1731971792906/Put/seqid=0 2024-11-18T23:16:34,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731971794170, max timestamp=1731971794208Num uniques 1031; 2024-11-18T23:16:34,220 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-18T23:16:34,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971794170, max timestamp=1731971794208Num uniques 1031; 2024-11-18T23:16:34,223 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=6870, suffix heap size=135192,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=150 2024-11-18T23:16:34,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971794170, max timestamp=1731971794208Num uniques 1031; 2024-11-18T23:16:34,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=6870, suffix heap size=135192,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=150 2024-11-18T23:16:34,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742120_1296 (size=12301) 2024-11-18T23:16:34,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=872 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/04df726e4e214aa981624f6af6a97cd4 2024-11-18T23:16:34,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/1dcde74ceb654de9a5838617a8656ebc as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1dcde74ceb654de9a5838617a8656ebc 2024-11-18T23:16:34,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731971794289, max timestamp=1731971794299Num uniques 1009; 2024-11-18T23:16:34,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8700, suffix heap size=132904,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=190 2024-11-18T23:16:34,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731971794289, max timestamp=1731971794299Num uniques 1009; 2024-11-18T23:16:34,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8700, suffix heap size=132904,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=190 2024-11-18T23:16:34,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731971794289, max timestamp=1731971794299Num uniques 1040; 2024-11-18T23:16:34,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-18T23:16:34,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1dcde74ceb654de9a5838617a8656ebc, entries=150, sequenceid=872, filesize=12.0 K 2024-11-18T23:16:34,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/b355d9b474e34bb3a33d447dbbe4e0d8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/b355d9b474e34bb3a33d447dbbe4e0d8 2024-11-18T23:16:34,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/b355d9b474e34bb3a33d447dbbe4e0d8, entries=150, sequenceid=872, filesize=12.0 K 2024-11-18T23:16:34,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/04df726e4e214aa981624f6af6a97cd4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04df726e4e214aa981624f6af6a97cd4 2024-11-18T23:16:34,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04df726e4e214aa981624f6af6a97cd4, entries=150, sequenceid=872, filesize=12.0 K 2024-11-18T23:16:34,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.15 KB/33945, heapSize ~85.20 KB/87240, currentSize=127.47 KB/130530 for d0aae39639c33cd55093c78ecbe9c4f6 in 842ms, sequenceid=872, compaction requested=false 2024-11-18T23:16:34,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:34,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=134.18 KB heapSize=382.08 KB 2024-11-18T23:16:34,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:16:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:16:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:34,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:16:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971854429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971854433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/e391b5b44a714e0a95e8b1b17e5e4fc0 is 50, key is test_row_0/A:col10/1731971794401/Put/seqid=0 2024-11-18T23:16:34,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971854438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971854441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971854437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742121_1297 (size=12301) 2024-11-18T23:16:34,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1067 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/e391b5b44a714e0a95e8b1b17e5e4fc0 2024-11-18T23:16:34,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971854557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971854558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/2fedd4b6cab84fa48eebb6217fa40233 is 50, key is test_row_0/B:col10/1731971794401/Put/seqid=0 2024-11-18T23:16:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971854569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971854568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971854569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742122_1298 (size=12301) 2024-11-18T23:16:34,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971854778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971854778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,782 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971854798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971854798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971854798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:35,004 INFO [Thread-991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-991. 2024-11-18T23:16:35,004 DEBUG [Thread-991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,004 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,005 DEBUG [Thread-991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,006 INFO [Thread-985 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-985. 2024-11-18T23:16:35,006 DEBUG [Thread-985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,006 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,006 DEBUG [Thread-985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,009 INFO [Thread-983 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-983. 2024-11-18T23:16:35,009 DEBUG [Thread-983 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,009 DEBUG [Thread-983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,009 DEBUG [Thread-983 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,011 INFO [Thread-989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-989. 2024-11-18T23:16:35,011 DEBUG [Thread-989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,011 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,011 DEBUG [Thread-989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,012 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,012 INFO [Thread-987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-987. 2024-11-18T23:16:35,012 DEBUG [Thread-987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,012 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,013 DEBUG [Thread-987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1067 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/2fedd4b6cab84fa48eebb6217fa40233 2024-11-18T23:16:35,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b90a09d0ec3743ce961a9cfce12a2c46 is 50, key is test_row_0/C:col10/1731971794401/Put/seqid=0 2024-11-18T23:16:35,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742123_1299 (size=12301) 2024-11-18T23:16:35,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=1067 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b90a09d0ec3743ce961a9cfce12a2c46 2024-11-18T23:16:35,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48772 deadline: 1731971855087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:35,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48762 deadline: 1731971855087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:35,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/e391b5b44a714e0a95e8b1b17e5e4fc0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/e391b5b44a714e0a95e8b1b17e5e4fc0 2024-11-18T23:16:35,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:35,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/e391b5b44a714e0a95e8b1b17e5e4fc0, entries=150, sequenceid=1067, filesize=12.0 K 2024-11-18T23:16:35,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/2fedd4b6cab84fa48eebb6217fa40233 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/2fedd4b6cab84fa48eebb6217fa40233 2024-11-18T23:16:35,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:35,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731971855120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,121 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:35,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/2fedd4b6cab84fa48eebb6217fa40233, entries=150, sequenceid=1067, filesize=12.0 K 2024-11-18T23:16:35,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:35,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48804 deadline: 1731971855122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:35,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/b90a09d0ec3743ce961a9cfce12a2c46 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b90a09d0ec3743ce961a9cfce12a2c46 2024-11-18T23:16:35,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48764 deadline: 1731971855122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:35,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:16:35,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b90a09d0ec3743ce961a9cfce12a2c46, entries=150, sequenceid=1067, filesize=12.0 K 2024-11-18T23:16:35,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~402.66 KB/412320, currentSize=40.25 KB/41220 for d0aae39639c33cd55093c78ecbe9c4f6 in 734ms, sequenceid=1067, compaction requested=true 2024-11-18T23:16:35,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:35,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:35,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:35,143 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:35,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:35,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:35,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0aae39639c33cd55093c78ecbe9c4f6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:35,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:35,144 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:35,144 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/A is initiating minor compaction (all files) 2024-11-18T23:16:35,144 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/A in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:35,144 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/86e807558158421c9429cff1e8e623a4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1dcde74ceb654de9a5838617a8656ebc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/e391b5b44a714e0a95e8b1b17e5e4fc0] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=37.0 K 2024-11-18T23:16:35,145 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86e807558158421c9429cff1e8e623a4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731971792185 2024-11-18T23:16:35,146 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dcde74ceb654de9a5838617a8656ebc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1731971792906 2024-11-18T23:16:35,147 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e391b5b44a714e0a95e8b1b17e5e4fc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1067, earliestPutTs=1731971794369 2024-11-18T23:16:35,148 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:35,160 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:35,160 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/B is initiating minor compaction (all files) 2024-11-18T23:16:35,160 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/B in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:35,161 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f2c3f20127214d6eb97264ba4c64fd07, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/b355d9b474e34bb3a33d447dbbe4e0d8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/2fedd4b6cab84fa48eebb6217fa40233] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=37.0 K 2024-11-18T23:16:35,168 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c3f20127214d6eb97264ba4c64fd07, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731971792185 2024-11-18T23:16:35,176 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting b355d9b474e34bb3a33d447dbbe4e0d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1731971792906 2024-11-18T23:16:35,184 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fedd4b6cab84fa48eebb6217fa40233, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1067, earliestPutTs=1731971794369 2024-11-18T23:16:35,198 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#A#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:35,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/1df8a53680244de4803c3e6026431937 is 50, key is test_row_0/A:col10/1731971794401/Put/seqid=0 2024-11-18T23:16:35,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742124_1300 (size=13391) 2024-11-18T23:16:35,255 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#B#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:35,255 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/839b338230294c218a1f627bc96e9032 is 50, key is test_row_0/B:col10/1731971794401/Put/seqid=0 2024-11-18T23:16:35,255 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/1df8a53680244de4803c3e6026431937 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1df8a53680244de4803c3e6026431937 2024-11-18T23:16:35,305 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/A of d0aae39639c33cd55093c78ecbe9c4f6 into 1df8a53680244de4803c3e6026431937(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:35,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:35,305 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/A, priority=13, startTime=1731971795143; duration=0sec 2024-11-18T23:16:35,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:35,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:A 2024-11-18T23:16:35,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:35,313 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:35,313 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): d0aae39639c33cd55093c78ecbe9c4f6/C is initiating minor compaction (all files) 2024-11-18T23:16:35,314 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0aae39639c33cd55093c78ecbe9c4f6/C in TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:35,314 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/665f8bb5920645ea9b90864c3db710e8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04df726e4e214aa981624f6af6a97cd4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b90a09d0ec3743ce961a9cfce12a2c46] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp, totalSize=37.0 K 2024-11-18T23:16:35,315 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665f8bb5920645ea9b90864c3db710e8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=850, earliestPutTs=1731971792185 2024-11-18T23:16:35,316 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04df726e4e214aa981624f6af6a97cd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=872, earliestPutTs=1731971792906 2024-11-18T23:16:35,316 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b90a09d0ec3743ce961a9cfce12a2c46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1067, earliestPutTs=1731971794369 2024-11-18T23:16:35,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742125_1301 (size=13391) 2024-11-18T23:16:35,354 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0aae39639c33cd55093c78ecbe9c4f6#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:35,355 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/139fdd270f204456b91c70dfb67920ca is 50, key is test_row_0/C:col10/1731971794401/Put/seqid=0 2024-11-18T23:16:35,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742126_1302 (size=13391) 2024-11-18T23:16:35,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:35,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:16:35,600 INFO [Thread-972 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-972. 2024-11-18T23:16:35,601 DEBUG [Thread-972 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,601 DEBUG [Thread-972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:35,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:35,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:35,601 DEBUG [Thread-972 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:35,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:35,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:35,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,601 INFO [Thread-974 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-974. 2024-11-18T23:16:35,601 DEBUG [Thread-974 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,601 DEBUG [Thread-974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,601 DEBUG [Thread-974 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/768efd343e054458afccf46fca9d49dd is 50, key is test_row_0/A:col10/1731971794434/Put/seqid=0 2024-11-18T23:16:35,631 INFO [Thread-976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-976. 2024-11-18T23:16:35,631 INFO [Thread-978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-978. 2024-11-18T23:16:35,631 DEBUG [Thread-976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,631 DEBUG [Thread-978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,631 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,631 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,631 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,631 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,638 INFO [Thread-980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-980. 2024-11-18T23:16:35,638 DEBUG [Thread-980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:16:35,638 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,639 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742127_1303 (size=12301) 2024-11-18T23:16:35,777 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/839b338230294c218a1f627bc96e9032 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/839b338230294c218a1f627bc96e9032 2024-11-18T23:16:35,817 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/B of d0aae39639c33cd55093c78ecbe9c4f6 into 839b338230294c218a1f627bc96e9032(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:35,817 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:35,817 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/B, priority=13, startTime=1731971795143; duration=0sec 2024-11-18T23:16:35,817 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:35,817 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:B 2024-11-18T23:16:35,829 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/139fdd270f204456b91c70dfb67920ca as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/139fdd270f204456b91c70dfb67920ca 2024-11-18T23:16:35,845 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0aae39639c33cd55093c78ecbe9c4f6/C of d0aae39639c33cd55093c78ecbe9c4f6 into 139fdd270f204456b91c70dfb67920ca(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:35,845 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:35,845 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., storeName=d0aae39639c33cd55093c78ecbe9c4f6/C, priority=13, startTime=1731971795143; duration=0sec 2024-11-18T23:16:35,845 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:35,845 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0aae39639c33cd55093c78ecbe9c4f6:C 2024-11-18T23:16:35,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-18T23:16:35,948 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 197 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 170 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 193 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 210 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 211 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T23:16:35,948 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1428 2024-11-18T23:16:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1464 2024-11-18T23:16:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1423 2024-11-18T23:16:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1451 2024-11-18T23:16:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1460 2024-11-18T23:16:35,949 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T23:16:35,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:16:35,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:16:35,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:35,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:16:35,953 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T23:16:35,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T23:16:35,964 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971795964"}]},"ts":"1731971795964"} 2024-11-18T23:16:35,966 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T23:16:35,966 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T23:16:35,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:16:35,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, UNASSIGN}] 2024-11-18T23:16:35,971 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, UNASSIGN 2024-11-18T23:16:35,972 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=d0aae39639c33cd55093c78ecbe9c4f6, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:35,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=71, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, UNASSIGN because future has completed 2024-11-18T23:16:35,976 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:16:35,976 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:16:36,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1079 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/768efd343e054458afccf46fca9d49dd 2024-11-18T23:16:36,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T23:16:36,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/af700c6d3fd14130acd895fcce05950b is 50, key is test_row_0/B:col10/1731971794434/Put/seqid=0 2024-11-18T23:16:36,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742128_1304 (size=12301) 2024-11-18T23:16:36,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1079 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/af700c6d3fd14130acd895fcce05950b 2024-11-18T23:16:36,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/18b8c8821a6c4583a4b83f98af9a55c1 is 50, key is test_row_0/C:col10/1731971794434/Put/seqid=0 2024-11-18T23:16:36,139 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(122): Close d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:36,139 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:16:36,139 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1722): Closing d0aae39639c33cd55093c78ecbe9c4f6, disabling compactions & flushes 2024-11-18T23:16:36,139 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:36,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742129_1305 (size=12301) 2024-11-18T23:16:36,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1079 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/18b8c8821a6c4583a4b83f98af9a55c1 2024-11-18T23:16:36,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/768efd343e054458afccf46fca9d49dd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/768efd343e054458afccf46fca9d49dd 2024-11-18T23:16:36,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/768efd343e054458afccf46fca9d49dd, entries=150, sequenceid=1079, filesize=12.0 K 2024-11-18T23:16:36,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/af700c6d3fd14130acd895fcce05950b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/af700c6d3fd14130acd895fcce05950b 2024-11-18T23:16:36,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/af700c6d3fd14130acd895fcce05950b, entries=150, sequenceid=1079, filesize=12.0 K 2024-11-18T23:16:36,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/18b8c8821a6c4583a4b83f98af9a55c1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/18b8c8821a6c4583a4b83f98af9a55c1 2024-11-18T23:16:36,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/18b8c8821a6c4583a4b83f98af9a55c1, entries=150, sequenceid=1079, filesize=12.0 K 2024-11-18T23:16:36,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for d0aae39639c33cd55093c78ecbe9c4f6 in 662ms, sequenceid=1079, compaction requested=false 2024-11-18T23:16:36,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0aae39639c33cd55093c78ecbe9c4f6: 2024-11-18T23:16:36,263 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:36,263 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:36,263 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. after waiting 0 ms 2024-11-18T23:16:36,263 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:36,263 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(2902): Flushing d0aae39639c33cd55093c78ecbe9c4f6 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T23:16:36,263 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=A 2024-11-18T23:16:36,263 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:36,264 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=B 2024-11-18T23:16:36,264 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:36,264 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0aae39639c33cd55093c78ecbe9c4f6, store=C 2024-11-18T23:16:36,264 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T23:16:36,282 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/494c91a74445452695a01daadfda5dda is 50, key is test_row_1/A:col10/1731971795637/Put/seqid=0 2024-11-18T23:16:36,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742130_1306 (size=9857) 2024-11-18T23:16:36,316 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=1087 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/494c91a74445452695a01daadfda5dda 2024-11-18T23:16:36,327 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/ec75a30b22294c9b84f695267fd2e924 is 50, key is test_row_1/B:col10/1731971795637/Put/seqid=0 2024-11-18T23:16:36,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742131_1307 (size=9857) 2024-11-18T23:16:36,377 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=1087 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/ec75a30b22294c9b84f695267fd2e924 2024-11-18T23:16:36,402 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9d7172ac371a482c8dcea7eba728ec52 is 50, key is test_row_1/C:col10/1731971795637/Put/seqid=0 2024-11-18T23:16:36,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742132_1308 (size=9857) 2024-11-18T23:16:36,455 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=1087 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9d7172ac371a482c8dcea7eba728ec52 2024-11-18T23:16:36,464 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/A/494c91a74445452695a01daadfda5dda as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/494c91a74445452695a01daadfda5dda 2024-11-18T23:16:36,471 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/494c91a74445452695a01daadfda5dda, entries=100, sequenceid=1087, filesize=9.6 K 2024-11-18T23:16:36,472 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/B/ec75a30b22294c9b84f695267fd2e924 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ec75a30b22294c9b84f695267fd2e924 2024-11-18T23:16:36,484 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ec75a30b22294c9b84f695267fd2e924, entries=100, sequenceid=1087, filesize=9.6 K 2024-11-18T23:16:36,485 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/.tmp/C/9d7172ac371a482c8dcea7eba728ec52 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9d7172ac371a482c8dcea7eba728ec52 2024-11-18T23:16:36,509 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9d7172ac371a482c8dcea7eba728ec52, entries=100, sequenceid=1087, filesize=9.6 K 2024-11-18T23:16:36,514 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d0aae39639c33cd55093c78ecbe9c4f6 in 251ms, sequenceid=1087, compaction requested=true 2024-11-18T23:16:36,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/cffc58f177714eb9b97ca6a89050edd4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6428d16ffd1948f4987bc0d521b4cb6e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9515bbc9abec458d8f229332b54df9a8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/914cb2a7c57848ccbcb4900febbfab4a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f521e8f6026b4e8184cfea8a4dc33dd9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/df87b5504ab2419bb1043163875828c7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/c79defc0577f457e9fe0a81b21cf64f9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d4f275c7344f40ef829a9a6765f94837, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/87f2f410db7f46a39a363d47044e86f5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/63de77569f0440bebd9994871035adfd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6946515d028742d0b3107f3ecb3592a7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/33cb6e71393b4388af2f7b5dc08ce500, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/5c4afd79f3a14b678e14497e0bd6d24a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d3d7c2841ee64b23ab2026f9f4e173d2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4f512133578944708c7c9ca798371c31, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77c4eae1ff2649db9652a58b8d296845, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/14e1ef6474e54411951286099063d259, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/159ad81677eb420f9933f71e56a56b8a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d9c9e7c2eb8240169a1951ae9580ee08, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/2a8ce31c803d4e659cda3343a5fb9d25, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/274c976163204170be23c0110ba3d061, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4617b4b20ff6426a92683677fe4c8d24, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/3b7642bc292d4e5396519d5445db46a1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/8c73240a4f4544a3a1b59f6540467af2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f84154e71f634daa852f5c141c472727, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/a7d8c84661da4e7c8384e6eb520a8b48, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9bd49c7e9d314df7adf104be815e477f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/deb87bceafe64223b3b0f942171d7912, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/86e807558158421c9429cff1e8e623a4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77af5f5b5ecc4d2ab26a7be8e355d1b6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1dcde74ceb654de9a5838617a8656ebc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/e391b5b44a714e0a95e8b1b17e5e4fc0] to archive 2024-11-18T23:16:36,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:16:36,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/cffc58f177714eb9b97ca6a89050edd4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/cffc58f177714eb9b97ca6a89050edd4 2024-11-18T23:16:36,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6428d16ffd1948f4987bc0d521b4cb6e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6428d16ffd1948f4987bc0d521b4cb6e 2024-11-18T23:16:36,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9515bbc9abec458d8f229332b54df9a8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9515bbc9abec458d8f229332b54df9a8 2024-11-18T23:16:36,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/914cb2a7c57848ccbcb4900febbfab4a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/914cb2a7c57848ccbcb4900febbfab4a 2024-11-18T23:16:36,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f521e8f6026b4e8184cfea8a4dc33dd9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f521e8f6026b4e8184cfea8a4dc33dd9 2024-11-18T23:16:36,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/df87b5504ab2419bb1043163875828c7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/df87b5504ab2419bb1043163875828c7 2024-11-18T23:16:36,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/c79defc0577f457e9fe0a81b21cf64f9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/c79defc0577f457e9fe0a81b21cf64f9 2024-11-18T23:16:36,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d4f275c7344f40ef829a9a6765f94837 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d4f275c7344f40ef829a9a6765f94837 2024-11-18T23:16:36,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/87f2f410db7f46a39a363d47044e86f5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/87f2f410db7f46a39a363d47044e86f5 2024-11-18T23:16:36,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/63de77569f0440bebd9994871035adfd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/63de77569f0440bebd9994871035adfd 2024-11-18T23:16:36,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6946515d028742d0b3107f3ecb3592a7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/6946515d028742d0b3107f3ecb3592a7 2024-11-18T23:16:36,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/33cb6e71393b4388af2f7b5dc08ce500 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/33cb6e71393b4388af2f7b5dc08ce500 2024-11-18T23:16:36,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/5c4afd79f3a14b678e14497e0bd6d24a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/5c4afd79f3a14b678e14497e0bd6d24a 2024-11-18T23:16:36,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d3d7c2841ee64b23ab2026f9f4e173d2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d3d7c2841ee64b23ab2026f9f4e173d2 2024-11-18T23:16:36,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4f512133578944708c7c9ca798371c31 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4f512133578944708c7c9ca798371c31 2024-11-18T23:16:36,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77c4eae1ff2649db9652a58b8d296845 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77c4eae1ff2649db9652a58b8d296845 2024-11-18T23:16:36,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/14e1ef6474e54411951286099063d259 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/14e1ef6474e54411951286099063d259 2024-11-18T23:16:36,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/159ad81677eb420f9933f71e56a56b8a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/159ad81677eb420f9933f71e56a56b8a 2024-11-18T23:16:36,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d9c9e7c2eb8240169a1951ae9580ee08 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/d9c9e7c2eb8240169a1951ae9580ee08 2024-11-18T23:16:36,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T23:16:36,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/2a8ce31c803d4e659cda3343a5fb9d25 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/2a8ce31c803d4e659cda3343a5fb9d25 2024-11-18T23:16:36,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/274c976163204170be23c0110ba3d061 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/274c976163204170be23c0110ba3d061 2024-11-18T23:16:36,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4617b4b20ff6426a92683677fe4c8d24 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/4617b4b20ff6426a92683677fe4c8d24 2024-11-18T23:16:36,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/3b7642bc292d4e5396519d5445db46a1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/3b7642bc292d4e5396519d5445db46a1 2024-11-18T23:16:36,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/8c73240a4f4544a3a1b59f6540467af2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/8c73240a4f4544a3a1b59f6540467af2 2024-11-18T23:16:36,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f84154e71f634daa852f5c141c472727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/f84154e71f634daa852f5c141c472727 2024-11-18T23:16:36,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/a7d8c84661da4e7c8384e6eb520a8b48 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/a7d8c84661da4e7c8384e6eb520a8b48 2024-11-18T23:16:36,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9bd49c7e9d314df7adf104be815e477f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/9bd49c7e9d314df7adf104be815e477f 2024-11-18T23:16:36,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/deb87bceafe64223b3b0f942171d7912 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/deb87bceafe64223b3b0f942171d7912 2024-11-18T23:16:36,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/86e807558158421c9429cff1e8e623a4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/86e807558158421c9429cff1e8e623a4 2024-11-18T23:16:36,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77af5f5b5ecc4d2ab26a7be8e355d1b6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/77af5f5b5ecc4d2ab26a7be8e355d1b6 2024-11-18T23:16:36,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1dcde74ceb654de9a5838617a8656ebc to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1dcde74ceb654de9a5838617a8656ebc 2024-11-18T23:16:36,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/e391b5b44a714e0a95e8b1b17e5e4fc0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/e391b5b44a714e0a95e8b1b17e5e4fc0 2024-11-18T23:16:36,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/89c072ab73964c3d828c1761dcffb6b9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e9523efa1ef942bd8a094bab402345d2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7747544eebc947969162c895dbd865e3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c86f371e4c1c49229ac3138717444905, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/737a746a1ac04648b233954736fa63d1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/5aae61d7135f4b96882b812110962279, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7c753b7a02f54f3c90a69d2c2b194ce8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/20d70f64723c465fbd5debdc04e0dd82, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/3176ed32b5fa41fd950780c1a03edbf4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f767bde680d246ff94ee0d9484092976, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dd47ce58ca494922ba15828448f4864f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f750e4da358e4df3a785deb1f6c928b8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/8313cf15935e4df39d6cb1f7f9dcc93e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/be898f4c67604d56a45a7c7a43b22827, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/4a0f9a21f58d4099b347fae53a7fda4a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/14069a51a27e44928641c0b103dc05b1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/64dcf323e39b45a7aa060b7526df85c1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/734491cbbd7343eeb07c8ae313d8df4b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e67b30829fc7442394a4454d6fe2b6e3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dda5bd96336741c08aa5c0afd862a5d5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/9bf3430de81943878fdb230a00fe756e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/04022dceea364b1290c36093efcb5912, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/83d71311cc4949ca84c90659c12ca368, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c074b6fe2248496289d0b90f9a606598, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ac6bb53a93754e46909c067af4a3d805, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/aed0ffab56404a0eb2d211214bf59c4b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/d1dab36e26914e869e94becd8ced14ef, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/79458eefd4d9483ea1a18587b4eb9d62, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f2c3f20127214d6eb97264ba4c64fd07, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/fede8d7a0198403fa286d7ca322a996b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/b355d9b474e34bb3a33d447dbbe4e0d8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/2fedd4b6cab84fa48eebb6217fa40233] to archive 2024-11-18T23:16:36,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:16:36,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/89c072ab73964c3d828c1761dcffb6b9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/89c072ab73964c3d828c1761dcffb6b9 2024-11-18T23:16:36,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e9523efa1ef942bd8a094bab402345d2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e9523efa1ef942bd8a094bab402345d2 2024-11-18T23:16:36,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7747544eebc947969162c895dbd865e3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7747544eebc947969162c895dbd865e3 2024-11-18T23:16:36,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c86f371e4c1c49229ac3138717444905 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c86f371e4c1c49229ac3138717444905 2024-11-18T23:16:36,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/737a746a1ac04648b233954736fa63d1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/737a746a1ac04648b233954736fa63d1 2024-11-18T23:16:36,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/5aae61d7135f4b96882b812110962279 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/5aae61d7135f4b96882b812110962279 2024-11-18T23:16:36,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7c753b7a02f54f3c90a69d2c2b194ce8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/7c753b7a02f54f3c90a69d2c2b194ce8 2024-11-18T23:16:36,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/20d70f64723c465fbd5debdc04e0dd82 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/20d70f64723c465fbd5debdc04e0dd82 2024-11-18T23:16:36,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/3176ed32b5fa41fd950780c1a03edbf4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/3176ed32b5fa41fd950780c1a03edbf4 2024-11-18T23:16:36,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f767bde680d246ff94ee0d9484092976 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f767bde680d246ff94ee0d9484092976 2024-11-18T23:16:36,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dd47ce58ca494922ba15828448f4864f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dd47ce58ca494922ba15828448f4864f 2024-11-18T23:16:36,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f750e4da358e4df3a785deb1f6c928b8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f750e4da358e4df3a785deb1f6c928b8 2024-11-18T23:16:36,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/8313cf15935e4df39d6cb1f7f9dcc93e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/8313cf15935e4df39d6cb1f7f9dcc93e 2024-11-18T23:16:36,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/be898f4c67604d56a45a7c7a43b22827 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/be898f4c67604d56a45a7c7a43b22827 2024-11-18T23:16:36,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/4a0f9a21f58d4099b347fae53a7fda4a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/4a0f9a21f58d4099b347fae53a7fda4a 2024-11-18T23:16:36,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/14069a51a27e44928641c0b103dc05b1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/14069a51a27e44928641c0b103dc05b1 2024-11-18T23:16:36,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/64dcf323e39b45a7aa060b7526df85c1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/64dcf323e39b45a7aa060b7526df85c1 2024-11-18T23:16:36,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/734491cbbd7343eeb07c8ae313d8df4b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/734491cbbd7343eeb07c8ae313d8df4b 2024-11-18T23:16:36,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e67b30829fc7442394a4454d6fe2b6e3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/e67b30829fc7442394a4454d6fe2b6e3 2024-11-18T23:16:36,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dda5bd96336741c08aa5c0afd862a5d5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/dda5bd96336741c08aa5c0afd862a5d5 2024-11-18T23:16:36,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/9bf3430de81943878fdb230a00fe756e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/9bf3430de81943878fdb230a00fe756e 2024-11-18T23:16:36,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/04022dceea364b1290c36093efcb5912 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/04022dceea364b1290c36093efcb5912 2024-11-18T23:16:36,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/83d71311cc4949ca84c90659c12ca368 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/83d71311cc4949ca84c90659c12ca368 2024-11-18T23:16:36,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c074b6fe2248496289d0b90f9a606598 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/c074b6fe2248496289d0b90f9a606598 2024-11-18T23:16:36,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ac6bb53a93754e46909c067af4a3d805 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ac6bb53a93754e46909c067af4a3d805 2024-11-18T23:16:36,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/aed0ffab56404a0eb2d211214bf59c4b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/aed0ffab56404a0eb2d211214bf59c4b 2024-11-18T23:16:36,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/d1dab36e26914e869e94becd8ced14ef to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/d1dab36e26914e869e94becd8ced14ef 2024-11-18T23:16:36,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/79458eefd4d9483ea1a18587b4eb9d62 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/79458eefd4d9483ea1a18587b4eb9d62 2024-11-18T23:16:36,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f2c3f20127214d6eb97264ba4c64fd07 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/f2c3f20127214d6eb97264ba4c64fd07 2024-11-18T23:16:36,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/fede8d7a0198403fa286d7ca322a996b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/fede8d7a0198403fa286d7ca322a996b 2024-11-18T23:16:36,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/b355d9b474e34bb3a33d447dbbe4e0d8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/b355d9b474e34bb3a33d447dbbe4e0d8 2024-11-18T23:16:36,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/2fedd4b6cab84fa48eebb6217fa40233 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/2fedd4b6cab84fa48eebb6217fa40233 2024-11-18T23:16:37,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d36c21154bde438db8d9164148e2a8a8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04fdd87fd5474a2c8bc2a8ceedb325b6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b9c9ecb131674b589ec31e6c4e7cd5ef, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ded4d638590049faae7c7d7cc945352b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b1c886f278a6440ca2b3b9b5c0c5715e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c3fa4b00cd544c7eb191f24b72287b70, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/622706c549f148a9890341223175deb4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/efdfe68741aa41e28fb5e7a4bb0db4c5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b92fa8e84f934fe4ae36ee2fe150ff3a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ebaea5f1f7034671bdd03f1923765b1f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/cd4eb7978d9647edac99067dfba0f08c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/e290962ae96d41f785879936d7195d81, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/1ccf0b6c950f435daa01dd9518f873ad, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ae24754686f44c309c2fe3c0ab464d6d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9e500e7f4a8647b3b1868d785db42670, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a4b0320c428740f3ad7e27ba16605bdd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9470dac1c2824e0098ec3c754fdc85bd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8f07cdc0d464208b608784de8e15b76, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/7dd66fd9f45c48eab44e4b8ea5d3c64e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a0c54a78d1a140808d76defea9e9de76, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c67e9c21cf8f494caf4c1c32543e1b84, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8b8e7eb072945a0a2f703da038f1aa5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d671872c6f944b2b80475b8cc7e91648, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/2fb2e53addaf4e14a9df4412de5660c0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/16fded2395e747a7ad069faf3e790e41, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a9b68a972d884d39970004c798d09c5b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/5c71d06ccfd047b1b89e60e211b698ce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/421350a239244389addb2343578d342a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/665f8bb5920645ea9b90864c3db710e8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9473a981f7854754a1f39f4827d212f5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04df726e4e214aa981624f6af6a97cd4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b90a09d0ec3743ce961a9cfce12a2c46] to archive 2024-11-18T23:16:37,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:16:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d36c21154bde438db8d9164148e2a8a8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d36c21154bde438db8d9164148e2a8a8 2024-11-18T23:16:37,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04fdd87fd5474a2c8bc2a8ceedb325b6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04fdd87fd5474a2c8bc2a8ceedb325b6 2024-11-18T23:16:37,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b9c9ecb131674b589ec31e6c4e7cd5ef to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b9c9ecb131674b589ec31e6c4e7cd5ef 2024-11-18T23:16:37,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ded4d638590049faae7c7d7cc945352b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ded4d638590049faae7c7d7cc945352b 2024-11-18T23:16:37,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b1c886f278a6440ca2b3b9b5c0c5715e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b1c886f278a6440ca2b3b9b5c0c5715e 2024-11-18T23:16:37,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c3fa4b00cd544c7eb191f24b72287b70 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c3fa4b00cd544c7eb191f24b72287b70 2024-11-18T23:16:37,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/622706c549f148a9890341223175deb4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/622706c549f148a9890341223175deb4 2024-11-18T23:16:37,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/efdfe68741aa41e28fb5e7a4bb0db4c5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/efdfe68741aa41e28fb5e7a4bb0db4c5 2024-11-18T23:16:37,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b92fa8e84f934fe4ae36ee2fe150ff3a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b92fa8e84f934fe4ae36ee2fe150ff3a 2024-11-18T23:16:37,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ebaea5f1f7034671bdd03f1923765b1f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ebaea5f1f7034671bdd03f1923765b1f 2024-11-18T23:16:37,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/cd4eb7978d9647edac99067dfba0f08c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/cd4eb7978d9647edac99067dfba0f08c 2024-11-18T23:16:37,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/e290962ae96d41f785879936d7195d81 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/e290962ae96d41f785879936d7195d81 2024-11-18T23:16:37,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/1ccf0b6c950f435daa01dd9518f873ad to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/1ccf0b6c950f435daa01dd9518f873ad 2024-11-18T23:16:37,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ae24754686f44c309c2fe3c0ab464d6d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/ae24754686f44c309c2fe3c0ab464d6d 2024-11-18T23:16:37,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9e500e7f4a8647b3b1868d785db42670 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9e500e7f4a8647b3b1868d785db42670 2024-11-18T23:16:37,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a4b0320c428740f3ad7e27ba16605bdd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a4b0320c428740f3ad7e27ba16605bdd 2024-11-18T23:16:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T23:16:37,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9470dac1c2824e0098ec3c754fdc85bd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9470dac1c2824e0098ec3c754fdc85bd 2024-11-18T23:16:37,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8f07cdc0d464208b608784de8e15b76 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8f07cdc0d464208b608784de8e15b76 2024-11-18T23:16:37,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/7dd66fd9f45c48eab44e4b8ea5d3c64e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/7dd66fd9f45c48eab44e4b8ea5d3c64e 2024-11-18T23:16:37,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a0c54a78d1a140808d76defea9e9de76 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a0c54a78d1a140808d76defea9e9de76 2024-11-18T23:16:37,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c67e9c21cf8f494caf4c1c32543e1b84 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/c67e9c21cf8f494caf4c1c32543e1b84 2024-11-18T23:16:37,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8b8e7eb072945a0a2f703da038f1aa5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b8b8e7eb072945a0a2f703da038f1aa5 2024-11-18T23:16:37,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d671872c6f944b2b80475b8cc7e91648 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/d671872c6f944b2b80475b8cc7e91648 2024-11-18T23:16:37,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/2fb2e53addaf4e14a9df4412de5660c0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/2fb2e53addaf4e14a9df4412de5660c0 2024-11-18T23:16:37,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/16fded2395e747a7ad069faf3e790e41 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/16fded2395e747a7ad069faf3e790e41 2024-11-18T23:16:37,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a9b68a972d884d39970004c798d09c5b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/a9b68a972d884d39970004c798d09c5b 2024-11-18T23:16:37,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/5c71d06ccfd047b1b89e60e211b698ce to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/5c71d06ccfd047b1b89e60e211b698ce 2024-11-18T23:16:37,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/421350a239244389addb2343578d342a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/421350a239244389addb2343578d342a 2024-11-18T23:16:37,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/665f8bb5920645ea9b90864c3db710e8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/665f8bb5920645ea9b90864c3db710e8 2024-11-18T23:16:37,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9473a981f7854754a1f39f4827d212f5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9473a981f7854754a1f39f4827d212f5 2024-11-18T23:16:37,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04df726e4e214aa981624f6af6a97cd4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/04df726e4e214aa981624f6af6a97cd4 2024-11-18T23:16:37,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b90a09d0ec3743ce961a9cfce12a2c46 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/b90a09d0ec3743ce961a9cfce12a2c46 2024-11-18T23:16:37,161 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/recovered.edits/1090.seqid, newMaxSeqId=1090, maxSeqId=1 2024-11-18T23:16:37,161 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. 2024-11-18T23:16:37,161 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] regionserver.HRegion(1676): Region close journal for d0aae39639c33cd55093c78ecbe9c4f6: Waiting for close lock at 1731971796139Running coprocessor pre-close hooks at 1731971796139Disabling compacts and flushes for region at 1731971796139Disabling writes for close at 1731971796263 (+124 ms)Obtaining lock to block concurrent updates at 1731971796263Preparing flush snapshotting stores in d0aae39639c33cd55093c78ecbe9c4f6 at 1731971796263Finished memstore snapshotting TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731971796264 (+1 ms)Flushing stores of TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6. at 1731971796267 (+3 ms)Flushing d0aae39639c33cd55093c78ecbe9c4f6/A: creating writer at 1731971796267Flushing d0aae39639c33cd55093c78ecbe9c4f6/A: appending metadata at 1731971796281 (+14 ms)Flushing d0aae39639c33cd55093c78ecbe9c4f6/A: closing flushed file at 1731971796281Flushing d0aae39639c33cd55093c78ecbe9c4f6/B: creating writer at 1731971796322 (+41 ms)Flushing d0aae39639c33cd55093c78ecbe9c4f6/B: appending metadata at 1731971796327 (+5 ms)Flushing d0aae39639c33cd55093c78ecbe9c4f6/B: closing flushed file at 1731971796327Flushing d0aae39639c33cd55093c78ecbe9c4f6/C: creating writer at 1731971796385 (+58 ms)Flushing d0aae39639c33cd55093c78ecbe9c4f6/C: appending metadata at 1731971796401 (+16 ms)Flushing d0aae39639c33cd55093c78ecbe9c4f6/C: closing flushed file at 1731971796401Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a31eb97: reopening flushed file at 1731971796463 (+62 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70a6ee4f: reopening flushed file at 1731971796471 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73ce8077: reopening flushed file at 1731971796484 (+13 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for d0aae39639c33cd55093c78ecbe9c4f6 in 251ms, sequenceid=1087, compaction requested=true at 1731971796514 (+30 ms)Writing region close event to WAL at 1731971797152 (+638 ms)Running coprocessor post-close hooks at 1731971797161 (+9 ms)Closed at 1731971797161 2024-11-18T23:16:37,164 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=73}] handler.UnassignRegionHandler(157): Closed d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:37,164 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=d0aae39639c33cd55093c78ecbe9c4f6, regionState=CLOSED 2024-11-18T23:16:37,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=72, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:16:37,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-18T23:16:37,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseRegionProcedure d0aae39639c33cd55093c78ecbe9c4f6, server=9d96596e55a9,32899,1731971713394 in 1.1910 sec 2024-11-18T23:16:37,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-18T23:16:37,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2100 sec 2024-11-18T23:16:37,183 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971797183"}]},"ts":"1731971797183"} 2024-11-18T23:16:37,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-11-18T23:16:37,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0aae39639c33cd55093c78ecbe9c4f6, UNASSIGN in 1.2040 sec 2024-11-18T23:16:37,191 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T23:16:37,191 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T23:16:37,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2400 sec 2024-11-18T23:16:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-18T23:16:38,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:38,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T23:16:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:38,110 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=74, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:38,111 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=74, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T23:16:38,132 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:38,156 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/recovered.edits] 2024-11-18T23:16:38,184 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1df8a53680244de4803c3e6026431937 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/1df8a53680244de4803c3e6026431937 2024-11-18T23:16:38,189 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/494c91a74445452695a01daadfda5dda to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/494c91a74445452695a01daadfda5dda 2024-11-18T23:16:38,191 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/768efd343e054458afccf46fca9d49dd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/A/768efd343e054458afccf46fca9d49dd 2024-11-18T23:16:38,197 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/839b338230294c218a1f627bc96e9032 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/839b338230294c218a1f627bc96e9032 2024-11-18T23:16:38,199 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/af700c6d3fd14130acd895fcce05950b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/af700c6d3fd14130acd895fcce05950b 2024-11-18T23:16:38,205 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ec75a30b22294c9b84f695267fd2e924 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/B/ec75a30b22294c9b84f695267fd2e924 2024-11-18T23:16:38,208 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/139fdd270f204456b91c70dfb67920ca to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/139fdd270f204456b91c70dfb67920ca 2024-11-18T23:16:38,209 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/18b8c8821a6c4583a4b83f98af9a55c1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/18b8c8821a6c4583a4b83f98af9a55c1 2024-11-18T23:16:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T23:16:38,232 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9d7172ac371a482c8dcea7eba728ec52 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/C/9d7172ac371a482c8dcea7eba728ec52 2024-11-18T23:16:38,272 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/recovered.edits/1090.seqid to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6/recovered.edits/1090.seqid 2024-11-18T23:16:38,281 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/d0aae39639c33cd55093c78ecbe9c4f6 2024-11-18T23:16:38,281 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T23:16:38,305 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=74, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:38,310 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T23:16:38,314 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T23:16:38,319 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=74, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:38,319 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T23:16:38,319 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731971798319"}]},"ts":"9223372036854775807"} 2024-11-18T23:16:38,325 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T23:16:38,325 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d0aae39639c33cd55093c78ecbe9c4f6, NAME => 'TestAcidGuarantees,,1731971773781.d0aae39639c33cd55093c78ecbe9c4f6.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T23:16:38,325 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T23:16:38,325 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731971798325"}]},"ts":"9223372036854775807"} 2024-11-18T23:16:38,333 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T23:16:38,341 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=74, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:38,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 233 msec 2024-11-18T23:16:38,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-18T23:16:38,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T23:16:38,435 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:38,457 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 251), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1443 (was 1129) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7516 (was 6583) - AvailableMemoryMB LEAK? - 2024-11-18T23:16:38,488 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1443, ProcessCount=11, AvailableMemoryMB=7509 2024-11-18T23:16:38,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:16:38,498 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:16:38,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:38,502 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:16:38,502 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:38,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 75 2024-11-18T23:16:38,508 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:16:38,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-18T23:16:38,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742133_1309 (size=1061) 2024-11-18T23:16:38,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-18T23:16:38,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-18T23:16:38,977 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:16:39,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742134_1310 (size=53) 2024-11-18T23:16:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-18T23:16:39,416 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:16:39,417 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 49690c5aec791d09014e5d76c85e7360, disabling compactions & flushes 2024-11-18T23:16:39,417 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,417 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,417 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. after waiting 0 ms 2024-11-18T23:16:39,417 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,417 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,417 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 49690c5aec791d09014e5d76c85e7360: Waiting for close lock at 1731971799416Disabling compacts and flushes for region at 1731971799416Disabling writes for close at 1731971799417 (+1 ms)Writing region close event to WAL at 1731971799417Closed at 1731971799417 2024-11-18T23:16:39,419 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:16:39,420 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731971799419"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731971799419"}]},"ts":"1731971799419"} 2024-11-18T23:16:39,425 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:16:39,427 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:16:39,427 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971799427"}]},"ts":"1731971799427"} 2024-11-18T23:16:39,432 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T23:16:39,433 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, ASSIGN}] 2024-11-18T23:16:39,447 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, ASSIGN 2024-11-18T23:16:39,448 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, ASSIGN; state=OFFLINE, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=false 2024-11-18T23:16:39,601 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:39,603 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=76, ppid=75, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, ASSIGN because future has completed 2024-11-18T23:16:39,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:16:39,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-18T23:16:39,792 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,792 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:16:39,792 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,792 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:16:39,793 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,793 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,812 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,821 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:39,822 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49690c5aec791d09014e5d76c85e7360 columnFamilyName A 2024-11-18T23:16:39,822 DEBUG [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:39,825 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(327): Store=49690c5aec791d09014e5d76c85e7360/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:39,825 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,831 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:39,831 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49690c5aec791d09014e5d76c85e7360 columnFamilyName B 2024-11-18T23:16:39,831 DEBUG [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:39,832 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(327): Store=49690c5aec791d09014e5d76c85e7360/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:39,833 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,837 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:39,838 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49690c5aec791d09014e5d76c85e7360 columnFamilyName C 2024-11-18T23:16:39,838 DEBUG [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:39,838 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(327): Store=49690c5aec791d09014e5d76c85e7360/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:39,839 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,839 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,848 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,852 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,868 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,868 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,873 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:16:39,877 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,883 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:16:39,884 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 49690c5aec791d09014e5d76c85e7360; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67712480, jitterRate=0.008994579315185547}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:16:39,884 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:39,885 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 49690c5aec791d09014e5d76c85e7360: Running coprocessor pre-open hook at 1731971799793Writing region info on filesystem at 1731971799793Initializing all the Stores at 1731971799803 (+10 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971799803Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971799812 (+9 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971799812Cleaning up temporary data from old regions at 1731971799868 (+56 ms)Running coprocessor post-open hooks at 1731971799884 (+16 ms)Region opened successfully at 1731971799885 (+1 ms) 2024-11-18T23:16:39,886 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., pid=77, masterSystemTime=1731971799760 2024-11-18T23:16:39,887 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,887 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:39,888 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=76 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:39,897 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=77, ppid=76, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:16:39,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-18T23:16:39,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; OpenRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 in 295 msec 2024-11-18T23:16:39,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-11-18T23:16:39,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, ASSIGN in 467 msec 2024-11-18T23:16:39,904 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:16:39,904 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971799904"}]},"ts":"1731971799904"} 2024-11-18T23:16:39,906 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T23:16:39,907 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=75, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:16:39,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.4090 sec 2024-11-18T23:16:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-11-18T23:16:40,665 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:40,669 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:40,669 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:40,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:40,675 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d8bb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:40,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:40,684 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:40,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c662dc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:40,686 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:40,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:40,695 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:40,697 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,42339,1731971712494 2024-11-18T23:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63caac17 2024-11-18T23:16:40,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:16:40,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:16:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:16:40,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:16:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T23:16:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742135_1311 (size=1097) 2024-11-18T23:16:40,795 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T23:16:40,795 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T23:16:40,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:16:40,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, REOPEN/MOVE}] 2024-11-18T23:16:40,808 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, REOPEN/MOVE 2024-11-18T23:16:40,810 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:40,815 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42339 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=9d96596e55a9,32899,1731971713394, table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-18T23:16:40,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, REOPEN/MOVE because future has completed 2024-11-18T23:16:40,873 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:16:40,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:16:41,029 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,029 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:16:41,030 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 49690c5aec791d09014e5d76c85e7360, disabling compactions & flushes 2024-11-18T23:16:41,030 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,030 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,030 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. after waiting 0 ms 2024-11-18T23:16:41,030 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,062 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T23:16:41,063 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,063 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 49690c5aec791d09014e5d76c85e7360: Waiting for close lock at 1731971801029Running coprocessor pre-close hooks at 1731971801029Disabling compacts and flushes for region at 1731971801029Disabling writes for close at 1731971801030 (+1 ms)Writing region close event to WAL at 1731971801037 (+7 ms)Running coprocessor post-close hooks at 1731971801062 (+25 ms)Closed at 1731971801063 (+1 ms) 2024-11-18T23:16:41,063 WARN [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionServer(3299): Not adding moved region record: 49690c5aec791d09014e5d76c85e7360 to self. 2024-11-18T23:16:41,069 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,071 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=CLOSED 2024-11-18T23:16:41,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:16:41,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-18T23:16:41,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 in 209 msec 2024-11-18T23:16:41,085 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, REOPEN/MOVE; state=CLOSED, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=true 2024-11-18T23:16:41,236 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, REOPEN/MOVE because future has completed 2024-11-18T23:16:41,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:16:41,408 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,408 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:16:41,409 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,409 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:16:41,409 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,409 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,417 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,426 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:41,426 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49690c5aec791d09014e5d76c85e7360 columnFamilyName A 2024-11-18T23:16:41,428 DEBUG [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:41,430 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(327): Store=49690c5aec791d09014e5d76c85e7360/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:41,431 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,433 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:41,433 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49690c5aec791d09014e5d76c85e7360 columnFamilyName B 2024-11-18T23:16:41,433 DEBUG [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:41,434 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(327): Store=49690c5aec791d09014e5d76c85e7360/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:41,435 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,437 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:16:41,437 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49690c5aec791d09014e5d76c85e7360 columnFamilyName C 2024-11-18T23:16:41,437 DEBUG [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:41,438 INFO [StoreOpener-49690c5aec791d09014e5d76c85e7360-1 {}] regionserver.HStore(327): Store=49690c5aec791d09014e5d76c85e7360/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:16:41,438 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,438 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,439 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,442 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,443 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,443 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,443 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:16:41,450 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,451 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 49690c5aec791d09014e5d76c85e7360; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65294891, jitterRate=-0.027030304074287415}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:16:41,451 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,453 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 49690c5aec791d09014e5d76c85e7360: Running coprocessor pre-open hook at 1731971801409Writing region info on filesystem at 1731971801409Initializing all the Stores at 1731971801414 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971801414Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971801416 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971801416Cleaning up temporary data from old regions at 1731971801443 (+27 ms)Running coprocessor post-open hooks at 1731971801451 (+8 ms)Region opened successfully at 1731971801452 (+1 ms) 2024-11-18T23:16:41,456 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., pid=82, masterSystemTime=1731971801392 2024-11-18T23:16:41,466 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T23:16:41,478 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=OPEN, openSeqNum=5, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,478 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,478 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=80, state=RUNNABLE, hasLock=false; OpenRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:16:41,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=80 2024-11-18T23:16:41,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=80, state=SUCCESS, hasLock=false; OpenRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 in 246 msec 2024-11-18T23:16:41,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-18T23:16:41,499 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 694 msec 2024-11-18T23:16:41,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-18T23:16:41,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, REOPEN/MOVE in 685 msec 2024-11-18T23:16:41,503 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 798 msec 2024-11-18T23:16:41,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T23:16:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-18T23:16:41,608 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:41,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,612 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,614 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6081f862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,615 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,617 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c2da0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,622 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@573d32a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e1266b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,637 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,637 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,638 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d839f36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,640 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@648bc3e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,650 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bc0b6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,652 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@796cb60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,653 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,655 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@795fb471, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,659 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575dfb57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee82a7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,667 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e92c4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,674 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,675 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65bd4ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,677 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e78440e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,681 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,682 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1386e288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cf0474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,688 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,688 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b5bb9f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,690 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40064, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63f90317, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:16:41,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:16:41,706 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:16:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:16:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:16:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f6c2df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:16:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:16:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:16:41,709 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:16:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc1756f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:16:41,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:41,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-11-18T23:16:41,716 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:41,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-18T23:16:41,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:41,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:41,721 DEBUG [Thread-1448 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,726 DEBUG [Thread-1433 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,727 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,728 DEBUG [Thread-1450 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,734 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,736 DEBUG [Thread-1446 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,741 DEBUG [Thread-1444 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,741 DEBUG [Thread-1452 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,743 DEBUG [Thread-1452 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,743 DEBUG [Thread-1452 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,743 DEBUG [Thread-1441 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,748 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,750 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,756 DEBUG [Thread-1435 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,757 DEBUG [Thread-1439 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,760 DEBUG [Thread-1437 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:16:41,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,766 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:16:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:16:41,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:16:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:16:41,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:16:41,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:41,790 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:41,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-18T23:16:41,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118da9536d04bac4da9b520536b51afe8b9_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971801742/Put/seqid=0 2024-11-18T23:16:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971861844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971861850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-18T23:16:41,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:41,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:41,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971861855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742136_1312 (size=12154) 2024-11-18T23:16:41,874 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:41,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971861856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971861858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,879 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118da9536d04bac4da9b520536b51afe8b9_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118da9536d04bac4da9b520536b51afe8b9_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:41,881 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/39b67c8699524a9388457af4c0f988be, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:41,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/39b67c8699524a9388457af4c0f988be is 175, key is test_row_0/A:col10/1731971801742/Put/seqid=0 2024-11-18T23:16:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742137_1313 (size=30955) 2024-11-18T23:16:41,925 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/39b67c8699524a9388457af4c0f988be 2024-11-18T23:16:41,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971861967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971861967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/9ec5dfdc806649689a462431cd3c64be is 50, key is test_row_0/B:col10/1731971801742/Put/seqid=0 2024-11-18T23:16:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971861980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971861988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:41,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:41,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971861988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742138_1314 (size=12001) 2024-11-18T23:16:42,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/9ec5dfdc806649689a462431cd3c64be 2024-11-18T23:16:42,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-18T23:16:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:42,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:42,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3 is 50, key is test_row_0/C:col10/1731971801742/Put/seqid=0 2024-11-18T23:16:42,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-18T23:16:42,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742139_1315 (size=12001) 2024-11-18T23:16:42,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3 2024-11-18T23:16:42,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/39b67c8699524a9388457af4c0f988be as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/39b67c8699524a9388457af4c0f988be 2024-11-18T23:16:42,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/39b67c8699524a9388457af4c0f988be, entries=150, sequenceid=16, filesize=30.2 K 2024-11-18T23:16:42,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/9ec5dfdc806649689a462431cd3c64be as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9ec5dfdc806649689a462431cd3c64be 2024-11-18T23:16:42,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9ec5dfdc806649689a462431cd3c64be, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T23:16:42,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3 2024-11-18T23:16:42,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-18T23:16:42,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 49690c5aec791d09014e5d76c85e7360 in 306ms, sequenceid=16, compaction requested=false 2024-11-18T23:16:42,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:42,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-18T23:16:42,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:42,185 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T23:16:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:42,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:42,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:42,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:42,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111880b2850022664bebb5fbd5cd4d2a4778_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971801854/Put/seqid=0 2024-11-18T23:16:42,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971862216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971862216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971862227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971862228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971862232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,245 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742140_1316 (size=12154) 2024-11-18T23:16:42,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971862337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971862337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971862348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-18T23:16:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,357 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971862348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971862348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971862548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971862557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971862567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971862567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971862568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:42,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111880b2850022664bebb5fbd5cd4d2a4778_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111880b2850022664bebb5fbd5cd4d2a4778_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/eebd02957b7741eda312206f91509f89, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:42,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/eebd02957b7741eda312206f91509f89 is 175, key is test_row_0/A:col10/1731971801854/Put/seqid=0 2024-11-18T23:16:42,709 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:16:42,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742141_1317 (size=30955) 2024-11-18T23:16:42,747 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/eebd02957b7741eda312206f91509f89 2024-11-18T23:16:42,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/d426879c21274248b18e8c01684270e7 is 50, key is test_row_0/B:col10/1731971801854/Put/seqid=0 2024-11-18T23:16:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-18T23:16:42,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971862869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742142_1318 (size=12001) 2024-11-18T23:16:42,888 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/d426879c21274248b18e8c01684270e7 2024-11-18T23:16:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971862879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971862880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971862880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971862881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:42,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:42,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/4151ce68498546f399d5a0030ca461b6 is 50, key is test_row_0/C:col10/1731971801854/Put/seqid=0 2024-11-18T23:16:42,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742143_1319 (size=12001) 2024-11-18T23:16:42,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/4151ce68498546f399d5a0030ca461b6 2024-11-18T23:16:42,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/eebd02957b7741eda312206f91509f89 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eebd02957b7741eda312206f91509f89 2024-11-18T23:16:42,986 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eebd02957b7741eda312206f91509f89, entries=150, sequenceid=36, filesize=30.2 K 2024-11-18T23:16:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/d426879c21274248b18e8c01684270e7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d426879c21274248b18e8c01684270e7 2024-11-18T23:16:43,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d426879c21274248b18e8c01684270e7, entries=150, sequenceid=36, filesize=11.7 K 2024-11-18T23:16:43,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/4151ce68498546f399d5a0030ca461b6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/4151ce68498546f399d5a0030ca461b6 2024-11-18T23:16:43,015 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/4151ce68498546f399d5a0030ca461b6, entries=150, sequenceid=36, filesize=11.7 K 2024-11-18T23:16:43,016 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 49690c5aec791d09014e5d76c85e7360 in 831ms, sequenceid=36, compaction requested=false 2024-11-18T23:16:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:43,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-18T23:16:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-11-18T23:16:43,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-18T23:16:43,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3010 sec 2024-11-18T23:16:43,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 1.3140 sec 2024-11-18T23:16:43,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T23:16:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:43,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:43,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fcc79e999b9d4a2a8330a39858a6aedd_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971803392/Put/seqid=0 2024-11-18T23:16:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742144_1320 (size=14594) 2024-11-18T23:16:43,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971863549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971863557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971863604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971863613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971863615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971863720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971863729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971863760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971863761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971863761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-18T23:16:43,891 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:43,898 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:43,900 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:43,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-18T23:16:43,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-18T23:16:43,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:43,912 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118fcc79e999b9d4a2a8330a39858a6aedd_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fcc79e999b9d4a2a8330a39858a6aedd_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:43,914 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a8cc128be99d4856881bfc7b793f6a32, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:43,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a8cc128be99d4856881bfc7b793f6a32 is 175, key is test_row_0/A:col10/1731971803392/Put/seqid=0 2024-11-18T23:16:43,915 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:43,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:43,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742145_1321 (size=39549) 2024-11-18T23:16:43,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a8cc128be99d4856881bfc7b793f6a32 2024-11-18T23:16:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971863957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971863972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:43,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:43,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/1917c802ae39444c90b1bf6b292d2458 is 50, key is test_row_0/B:col10/1731971803392/Put/seqid=0 2024-11-18T23:16:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971863987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-18T23:16:44,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971863997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971864000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742146_1322 (size=12001) 2024-11-18T23:16:44,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/1917c802ae39444c90b1bf6b292d2458 2024-11-18T23:16:44,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/902b1760c39946f7aee978bc53a6db8f is 50, key is test_row_0/C:col10/1731971803392/Put/seqid=0 2024-11-18T23:16:44,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-18T23:16:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:44,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:44,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742147_1323 (size=12001) 2024-11-18T23:16:44,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/902b1760c39946f7aee978bc53a6db8f 2024-11-18T23:16:44,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a8cc128be99d4856881bfc7b793f6a32 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a8cc128be99d4856881bfc7b793f6a32 2024-11-18T23:16:44,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a8cc128be99d4856881bfc7b793f6a32, entries=200, sequenceid=50, filesize=38.6 K 2024-11-18T23:16:44,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/1917c802ae39444c90b1bf6b292d2458 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1917c802ae39444c90b1bf6b292d2458 2024-11-18T23:16:44,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1917c802ae39444c90b1bf6b292d2458, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T23:16:44,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/902b1760c39946f7aee978bc53a6db8f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/902b1760c39946f7aee978bc53a6db8f 2024-11-18T23:16:44,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/902b1760c39946f7aee978bc53a6db8f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T23:16:44,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 49690c5aec791d09014e5d76c85e7360 in 811ms, sequenceid=50, compaction requested=true 2024-11-18T23:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:44,205 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:44,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:44,206 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:44,207 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:44,207 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:44,208 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:16:44,208 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:16:44,208 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,208 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/39b67c8699524a9388457af4c0f988be, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eebd02957b7741eda312206f91509f89, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a8cc128be99d4856881bfc7b793f6a32] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=99.1 K 2024-11-18T23:16:44,208 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,208 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/39b67c8699524a9388457af4c0f988be, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eebd02957b7741eda312206f91509f89, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a8cc128be99d4856881bfc7b793f6a32] 2024-11-18T23:16:44,208 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,208 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9ec5dfdc806649689a462431cd3c64be, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d426879c21274248b18e8c01684270e7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1917c802ae39444c90b1bf6b292d2458] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=35.2 K 2024-11-18T23:16:44,208 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ec5dfdc806649689a462431cd3c64be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731971801742 2024-11-18T23:16:44,209 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 39b67c8699524a9388457af4c0f988be, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731971801742 2024-11-18T23:16:44,209 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d426879c21274248b18e8c01684270e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731971801848 2024-11-18T23:16:44,209 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting eebd02957b7741eda312206f91509f89, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731971801848 2024-11-18T23:16:44,209 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a8cc128be99d4856881bfc7b793f6a32, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971802191 2024-11-18T23:16:44,209 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1917c802ae39444c90b1bf6b292d2458, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971802213 2024-11-18T23:16:44,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-18T23:16:44,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-18T23:16:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,241 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T23:16:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:44,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:44,251 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:44,251 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:44,252 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/d67e21f7c8e749808d9ffee4a3841245 is 50, key is test_row_0/B:col10/1731971803392/Put/seqid=0 2024-11-18T23:16:44,265 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118785442881dc047d8a15795ddba1c5725_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:44,267 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118785442881dc047d8a15795ddba1c5725_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:44,268 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118785442881dc047d8a15795ddba1c5725_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:44,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:44,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742148_1324 (size=12104) 2024-11-18T23:16:44,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742149_1325 (size=4469) 2024-11-18T23:16:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189f32bf86b2b04cd5a2ac6f07fbf5a0d4_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971803611/Put/seqid=0 2024-11-18T23:16:44,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742150_1326 (size=12154) 2024-11-18T23:16:44,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:44,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971864371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971864378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,408 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189f32bf86b2b04cd5a2ac6f07fbf5a0d4_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189f32bf86b2b04cd5a2ac6f07fbf5a0d4_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:44,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/002616834d114e2d97388f61e814413b, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:44,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/002616834d114e2d97388f61e814413b is 175, key is test_row_0/A:col10/1731971803611/Put/seqid=0 2024-11-18T23:16:44,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971864379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971864380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971864381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742151_1327 (size=30955) 2024-11-18T23:16:44,469 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/002616834d114e2d97388f61e814413b 2024-11-18T23:16:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/1abc7ad6dacb4168ba3ddd43b2e94f90 is 50, key is test_row_0/B:col10/1731971803611/Put/seqid=0 2024-11-18T23:16:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971864512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971864518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742152_1328 (size=12001) 2024-11-18T23:16:44,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-18T23:16:44,550 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/1abc7ad6dacb4168ba3ddd43b2e94f90 2024-11-18T23:16:44,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971864527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971864527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971864527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a57a7ef645ac4eaeaeaddc5110e20e61 is 50, key is test_row_0/C:col10/1731971803611/Put/seqid=0 2024-11-18T23:16:44,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742153_1329 (size=12001) 2024-11-18T23:16:44,618 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a57a7ef645ac4eaeaeaddc5110e20e61 2024-11-18T23:16:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/002616834d114e2d97388f61e814413b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/002616834d114e2d97388f61e814413b 2024-11-18T23:16:44,651 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/002616834d114e2d97388f61e814413b, entries=150, sequenceid=68, filesize=30.2 K 2024-11-18T23:16:44,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/1abc7ad6dacb4168ba3ddd43b2e94f90 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1abc7ad6dacb4168ba3ddd43b2e94f90 2024-11-18T23:16:44,660 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1abc7ad6dacb4168ba3ddd43b2e94f90, entries=150, sequenceid=68, filesize=11.7 K 2024-11-18T23:16:44,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a57a7ef645ac4eaeaeaddc5110e20e61 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a57a7ef645ac4eaeaeaddc5110e20e61 2024-11-18T23:16:44,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a57a7ef645ac4eaeaeaddc5110e20e61, entries=150, sequenceid=68, filesize=11.7 K 2024-11-18T23:16:44,669 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 49690c5aec791d09014e5d76c85e7360 in 428ms, sequenceid=68, compaction requested=true 2024-11-18T23:16:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-18T23:16:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-18T23:16:44,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-18T23:16:44,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 760 msec 2024-11-18T23:16:44,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 780 msec 2024-11-18T23:16:44,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T23:16:44,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:44,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:44,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:44,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:44,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:44,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:44,776 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#279 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:44,776 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/fb71ea9a069540c58f678b3f112ab5f5 is 175, key is test_row_0/A:col10/1731971803392/Put/seqid=0 2024-11-18T23:16:44,782 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/d67e21f7c8e749808d9ffee4a3841245 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d67e21f7c8e749808d9ffee4a3841245 2024-11-18T23:16:44,800 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into d67e21f7c8e749808d9ffee4a3841245(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:44,800 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:44,800 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=13, startTime=1731971804205; duration=0sec 2024-11-18T23:16:44,800 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:44,800 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:16:44,800 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:44,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742154_1330 (size=31058) 2024-11-18T23:16:44,817 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:44,817 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:16:44,818 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:44,818 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/4151ce68498546f399d5a0030ca461b6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/902b1760c39946f7aee978bc53a6db8f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a57a7ef645ac4eaeaeaddc5110e20e61] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=46.9 K 2024-11-18T23:16:44,820 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a90e8e3fbe9440ab4cb6ab3a894d2d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731971801742 2024-11-18T23:16:44,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ebfc605f4181415b95a5adb5039f3a82_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971804376/Put/seqid=0 2024-11-18T23:16:44,831 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4151ce68498546f399d5a0030ca461b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731971801848 2024-11-18T23:16:44,833 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 902b1760c39946f7aee978bc53a6db8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971802213 2024-11-18T23:16:44,834 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting a57a7ef645ac4eaeaeaddc5110e20e61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731971803595 2024-11-18T23:16:44,841 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/fb71ea9a069540c58f678b3f112ab5f5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/fb71ea9a069540c58f678b3f112ab5f5 2024-11-18T23:16:44,857 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into fb71ea9a069540c58f678b3f112ab5f5(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:44,857 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:44,857 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=13, startTime=1731971804205; duration=0sec 2024-11-18T23:16:44,857 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:44,857 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:16:44,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742155_1331 (size=14594) 2024-11-18T23:16:44,885 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:44,890 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#284 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:44,891 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/7815fd8040b64774ae10d610520a4772 is 50, key is test_row_0/C:col10/1731971803611/Put/seqid=0 2024-11-18T23:16:44,903 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ebfc605f4181415b95a5adb5039f3a82_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ebfc605f4181415b95a5adb5039f3a82_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:44,907 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cbc014664a3940a7a95ba08e1cf431ea, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:44,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cbc014664a3940a7a95ba08e1cf431ea is 175, key is test_row_0/A:col10/1731971804376/Put/seqid=0 2024-11-18T23:16:44,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742156_1332 (size=12139) 2024-11-18T23:16:44,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971864896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971864913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971864918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971864932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,939 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742157_1333 (size=39549) 2024-11-18T23:16:44,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971864935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:44,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:44,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:44,978 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/7815fd8040b64774ae10d610520a4772 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7815fd8040b64774ae10d610520a4772 2024-11-18T23:16:44,979 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cbc014664a3940a7a95ba08e1cf431ea 2024-11-18T23:16:45,011 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into 7815fd8040b64774ae10d610520a4772(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:45,011 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:45,011 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=12, startTime=1731971804205; duration=0sec 2024-11-18T23:16:45,011 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:45,011 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:16:45,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/b83725d5474341da8db90bd14478ccb0 is 50, key is test_row_0/B:col10/1731971804376/Put/seqid=0 2024-11-18T23:16:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-18T23:16:45,060 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:45,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742158_1334 (size=12001) 2024-11-18T23:16:45,063 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-18T23:16:45,065 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T23:16:45,066 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:45,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:45,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971865051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971865051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971865051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971865051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971865080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T23:16:45,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:45,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971865278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971865278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971865280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971865280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971865308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:45,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:45,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T23:16:45,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/b83725d5474341da8db90bd14478ccb0 2024-11-18T23:16:45,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d64f8e77f3d240009a14e240cc5d4b83 is 50, key is test_row_0/C:col10/1731971804376/Put/seqid=0 2024-11-18T23:16:45,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742159_1335 (size=12001) 2024-11-18T23:16:45,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971865600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971865604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971865607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971865608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971865627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:45,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:45,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:45,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:45,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T23:16:45,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:45,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:45,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d64f8e77f3d240009a14e240cc5d4b83 2024-11-18T23:16:45,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cbc014664a3940a7a95ba08e1cf431ea as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cbc014664a3940a7a95ba08e1cf431ea 2024-11-18T23:16:45,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cbc014664a3940a7a95ba08e1cf431ea, entries=200, sequenceid=83, filesize=38.6 K 2024-11-18T23:16:45,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/b83725d5474341da8db90bd14478ccb0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/b83725d5474341da8db90bd14478ccb0 2024-11-18T23:16:45,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/b83725d5474341da8db90bd14478ccb0, entries=150, sequenceid=83, filesize=11.7 K 2024-11-18T23:16:45,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d64f8e77f3d240009a14e240cc5d4b83 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d64f8e77f3d240009a14e240cc5d4b83 2024-11-18T23:16:45,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d64f8e77f3d240009a14e240cc5d4b83, entries=150, sequenceid=83, filesize=11.7 K 2024-11-18T23:16:46,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,013 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 49690c5aec791d09014e5d76c85e7360 in 1254ms, sequenceid=83, compaction requested=true 2024-11-18T23:16:46,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:46,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:46,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:46,013 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:46,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:46,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:46,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:46,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:46,014 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:46,014 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:16:46,014 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,014 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/fb71ea9a069540c58f678b3f112ab5f5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/002616834d114e2d97388f61e814413b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cbc014664a3940a7a95ba08e1cf431ea] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=99.2 K 2024-11-18T23:16:46,014 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,015 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/fb71ea9a069540c58f678b3f112ab5f5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/002616834d114e2d97388f61e814413b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cbc014664a3940a7a95ba08e1cf431ea] 2024-11-18T23:16:46,015 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb71ea9a069540c58f678b3f112ab5f5, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971802213 2024-11-18T23:16:46,015 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 002616834d114e2d97388f61e814413b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731971803595 2024-11-18T23:16:46,015 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc014664a3940a7a95ba08e1cf431ea, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731971804369 2024-11-18T23:16:46,016 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:46,022 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:46,029 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:46,029 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:16:46,029 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,029 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d67e21f7c8e749808d9ffee4a3841245, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1abc7ad6dacb4168ba3ddd43b2e94f90, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/b83725d5474341da8db90bd14478ccb0] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=35.3 K 2024-11-18T23:16:46,029 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411185b8d9d0606db41eea455f0fce8ca5d2c_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:46,032 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411185b8d9d0606db41eea455f0fce8ca5d2c_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:46,033 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d67e21f7c8e749808d9ffee4a3841245, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971802213 2024-11-18T23:16:46,034 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 1abc7ad6dacb4168ba3ddd43b2e94f90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731971803595 2024-11-18T23:16:46,034 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting b83725d5474341da8db90bd14478ccb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731971804369 2024-11-18T23:16:46,034 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185b8d9d0606db41eea455f0fce8ca5d2c_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:46,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742160_1336 (size=4469) 2024-11-18T23:16:46,071 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:46,072 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/0f58b67b53a44bddb60c10e1313ab05e is 50, key is test_row_0/B:col10/1731971804376/Put/seqid=0 2024-11-18T23:16:46,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742161_1337 (size=12207) 2024-11-18T23:16:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:46,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T23:16:46,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:46,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:46,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:46,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:46,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:46,138 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:46,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:46,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f9a1ee0aeb7b4835b6173ef680be5230_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971806126/Put/seqid=0 2024-11-18T23:16:46,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971866156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971866160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971866162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971866172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971866174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T23:16:46,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742162_1338 (size=17034) 2024-11-18T23:16:46,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971866277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971866277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971866290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971866297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:46,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971866307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,321 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,461 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#287 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:46,462 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/2165274606294e00a55d54631baf27cd is 175, key is test_row_0/A:col10/1731971804376/Put/seqid=0 2024-11-18T23:16:46,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:46,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742163_1339 (size=31161) 2024-11-18T23:16:46,506 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/0f58b67b53a44bddb60c10e1313ab05e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/0f58b67b53a44bddb60c10e1313ab05e 2024-11-18T23:16:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971866497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971866500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,517 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/2165274606294e00a55d54631baf27cd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/2165274606294e00a55d54631baf27cd 2024-11-18T23:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,529 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into 0f58b67b53a44bddb60c10e1313ab05e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:46,529 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:46,529 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=13, startTime=1731971806013; duration=0sec 2024-11-18T23:16:46,529 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:46,529 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:16:46,529 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T23:16:46,530 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T23:16:46,530 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T23:16:46,530 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. because compaction request was cancelled 2024-11-18T23:16:46,530 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:16:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971866518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971866518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,536 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into 2165274606294e00a55d54631baf27cd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,536 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,536 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=13, startTime=1731971806013; duration=0sec 2024-11-18T23:16:46,536 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:46,537 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:16:46,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971866527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:46,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,624 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:46,629 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f9a1ee0aeb7b4835b6173ef680be5230_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9a1ee0aeb7b4835b6173ef680be5230_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:46,632 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3b405885b9904eaa8c0dcff36f6f7622, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:46,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3b405885b9904eaa8c0dcff36f6f7622 is 175, key is test_row_0/A:col10/1731971806126/Put/seqid=0 2024-11-18T23:16:46,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742164_1340 (size=48139) 2024-11-18T23:16:46,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971866828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971866828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971866840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971866847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971866847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:46,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:46,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:46,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:46,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3b405885b9904eaa8c0dcff36f6f7622 2024-11-18T23:16:47,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/8b597cdc6a5e46749438a58eb7ecf75d is 50, key is test_row_0/B:col10/1731971806126/Put/seqid=0 2024-11-18T23:16:47,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742165_1341 (size=12001) 2024-11-18T23:16:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T23:16:47,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:47,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:47,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971867347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:47,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971867348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,359 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:47,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971867357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:47,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971867377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:47,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971867378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,386 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:47,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:47,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:47,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/8b597cdc6a5e46749438a58eb7ecf75d 2024-11-18T23:16:47,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:47,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d2acc9c767ab4e52ba4f33c8fe6343b6 is 50, key is test_row_0/C:col10/1731971806126/Put/seqid=0 2024-11-18T23:16:47,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742166_1342 (size=12001) 2024-11-18T23:16:47,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d2acc9c767ab4e52ba4f33c8fe6343b6 2024-11-18T23:16:47,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3b405885b9904eaa8c0dcff36f6f7622 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3b405885b9904eaa8c0dcff36f6f7622 2024-11-18T23:16:47,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3b405885b9904eaa8c0dcff36f6f7622, entries=250, sequenceid=107, filesize=47.0 K 2024-11-18T23:16:47,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/8b597cdc6a5e46749438a58eb7ecf75d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/8b597cdc6a5e46749438a58eb7ecf75d 2024-11-18T23:16:47,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/8b597cdc6a5e46749438a58eb7ecf75d, entries=150, sequenceid=107, filesize=11.7 K 2024-11-18T23:16:47,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d2acc9c767ab4e52ba4f33c8fe6343b6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d2acc9c767ab4e52ba4f33c8fe6343b6 2024-11-18T23:16:47,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d2acc9c767ab4e52ba4f33c8fe6343b6, entries=150, sequenceid=107, filesize=11.7 K 2024-11-18T23:16:47,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 49690c5aec791d09014e5d76c85e7360 in 1514ms, sequenceid=107, compaction requested=true 2024-11-18T23:16:47,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:47,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:47,647 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T23:16:47,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:47,647 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-18T23:16:47,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:47,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:47,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:47,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:47,650 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T23:16:47,650 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T23:16:47,650 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. because compaction request was cancelled 2024-11-18T23:16:47,650 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:16:47,650 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:47,650 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-18T23:16:47,651 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-18T23:16:47,651 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. because compaction request was cancelled 2024-11-18T23:16:47,651 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:16:47,651 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:47,651 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:16:47,651 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,652 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7815fd8040b64774ae10d610520a4772, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d64f8e77f3d240009a14e240cc5d4b83, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d2acc9c767ab4e52ba4f33c8fe6343b6] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=35.3 K 2024-11-18T23:16:47,652 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7815fd8040b64774ae10d610520a4772, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731971803595 2024-11-18T23:16:47,660 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d64f8e77f3d240009a14e240cc5d4b83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731971804369 2024-11-18T23:16:47,660 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2acc9c767ab4e52ba4f33c8fe6343b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731971804903 2024-11-18T23:16:47,683 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:47,684 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/8b26eb7a49324ea5b0b84c3decb676ff is 50, key is test_row_0/C:col10/1731971806126/Put/seqid=0 2024-11-18T23:16:47,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742167_1343 (size=12241) 2024-11-18T23:16:47,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-18T23:16:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:47,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:16:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:47,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:47,743 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/8b26eb7a49324ea5b0b84c3decb676ff as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8b26eb7a49324ea5b0b84c3decb676ff 2024-11-18T23:16:47,755 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into 8b26eb7a49324ea5b0b84c3decb676ff(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:47,755 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:47,755 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=13, startTime=1731971807647; duration=0sec 2024-11-18T23:16:47,755 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:47,755 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:16:47,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f76b100eabe1465a8eedcc044272ff8c_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971806159/Put/seqid=0 2024-11-18T23:16:47,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742168_1344 (size=12154) 2024-11-18T23:16:47,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:47,820 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f76b100eabe1465a8eedcc044272ff8c_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f76b100eabe1465a8eedcc044272ff8c_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:47,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f3146502761b435db3b8058276a5ed6e, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:47,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f3146502761b435db3b8058276a5ed6e is 175, key is test_row_0/A:col10/1731971806159/Put/seqid=0 2024-11-18T23:16:47,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742169_1345 (size=30955) 2024-11-18T23:16:47,872 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f3146502761b435db3b8058276a5ed6e 2024-11-18T23:16:47,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e758373b5e1148d68ae1428d1d3dc938 is 50, key is test_row_0/B:col10/1731971806159/Put/seqid=0 2024-11-18T23:16:47,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742170_1346 (size=12001) 2024-11-18T23:16:47,981 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e758373b5e1148d68ae1428d1d3dc938 2024-11-18T23:16:47,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/5b596daa5779478d8202543dedf11701 is 50, key is test_row_0/C:col10/1731971806159/Put/seqid=0 2024-11-18T23:16:48,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742171_1347 (size=12001) 2024-11-18T23:16:48,042 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/5b596daa5779478d8202543dedf11701 2024-11-18T23:16:48,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f3146502761b435db3b8058276a5ed6e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f3146502761b435db3b8058276a5ed6e 2024-11-18T23:16:48,099 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f3146502761b435db3b8058276a5ed6e, entries=150, sequenceid=120, filesize=30.2 K 2024-11-18T23:16:48,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e758373b5e1148d68ae1428d1d3dc938 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e758373b5e1148d68ae1428d1d3dc938 2024-11-18T23:16:48,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,125 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e758373b5e1148d68ae1428d1d3dc938, entries=150, sequenceid=120, filesize=11.7 K 2024-11-18T23:16:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/5b596daa5779478d8202543dedf11701 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/5b596daa5779478d8202543dedf11701 2024-11-18T23:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,152 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/5b596daa5779478d8202543dedf11701, entries=150, sequenceid=120, filesize=11.7 K 2024-11-18T23:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,156 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 49690c5aec791d09014e5d76c85e7360 in 421ms, sequenceid=120, compaction requested=true 2024-11-18T23:16:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:48,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-18T23:16:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-18T23:16:48,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-18T23:16:48,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0910 sec 2024-11-18T23:16:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.0970 sec 2024-11-18T23:16:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:48,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:16:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182b0fbf2f76ce463abcb1659dcd119828_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971808560/Put/seqid=0 2024-11-18T23:16:48,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:48,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742172_1348 (size=14744) 2024-11-18T23:16:48,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971868616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971808607, max timestamp=1731971808628Num uniques 859; 2024-11-18T23:16:48,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:16:48,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971868626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,637 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971868651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971868651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971868652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971868738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971868748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971868767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971868767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971868787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,794 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971868957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,972 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:48,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971868967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:48,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:48,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971868987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971868987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,001 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411182b0fbf2f76ce463abcb1659dcd119828_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182b0fbf2f76ce463abcb1659dcd119828_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,002 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f96c46d15e464347a74a5f6673efd0e1, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:49,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f96c46d15e464347a74a5f6673efd0e1 is 175, key is test_row_0/A:col10/1731971808560/Put/seqid=0 2024-11-18T23:16:49,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742173_1349 (size=39699) 2024-11-18T23:16:49,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971868997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-18T23:16:49,238 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:49,242 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-18T23:16:49,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T23:16:49,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:49,248 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:49,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:49,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971869277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971869277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971869308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971869308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971869318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,331 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T23:16:49,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-18T23:16:49,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:49,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,415 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f96c46d15e464347a74a5f6673efd0e1 2024-11-18T23:16:49,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e3821fc2c4d943499e15205d57037404 is 50, key is test_row_0/B:col10/1731971808560/Put/seqid=0 2024-11-18T23:16:49,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742174_1350 (size=12101) 2024-11-18T23:16:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T23:16:49,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-18T23:16:49,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:49,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-18T23:16:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971869800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971869800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971869827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971869832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971869837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:49,849 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:49,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T23:16:49,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e3821fc2c4d943499e15205d57037404 2024-11-18T23:16:49,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-18T23:16:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:49,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:49,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/9f1dd569067842d6818e844b91cc28d4 is 50, key is test_row_0/C:col10/1731971808560/Put/seqid=0 2024-11-18T23:16:49,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742175_1351 (size=12101) 2024-11-18T23:16:49,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/9f1dd569067842d6818e844b91cc28d4 2024-11-18T23:16:50,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/f96c46d15e464347a74a5f6673efd0e1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f96c46d15e464347a74a5f6673efd0e1 2024-11-18T23:16:50,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-18T23:16:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:50,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:50,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f96c46d15e464347a74a5f6673efd0e1, entries=200, sequenceid=131, filesize=38.8 K 2024-11-18T23:16:50,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e3821fc2c4d943499e15205d57037404 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e3821fc2c4d943499e15205d57037404 2024-11-18T23:16:50,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e3821fc2c4d943499e15205d57037404, entries=150, sequenceid=131, filesize=11.8 K 2024-11-18T23:16:50,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/9f1dd569067842d6818e844b91cc28d4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/9f1dd569067842d6818e844b91cc28d4 2024-11-18T23:16:50,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/9f1dd569067842d6818e844b91cc28d4, entries=150, sequenceid=131, filesize=11.8 K 2024-11-18T23:16:50,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 49690c5aec791d09014e5d76c85e7360 in 1632ms, sequenceid=131, compaction requested=true 2024-11-18T23:16:50,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:50,193 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:50,195 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:50,197 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:16:50,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48310 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:50,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:16:50,199 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,199 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/0f58b67b53a44bddb60c10e1313ab05e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/8b597cdc6a5e46749438a58eb7ecf75d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e758373b5e1148d68ae1428d1d3dc938, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e3821fc2c4d943499e15205d57037404] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=47.2 K 2024-11-18T23:16:50,200 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f58b67b53a44bddb60c10e1313ab05e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731971804369 2024-11-18T23:16:50,200 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b597cdc6a5e46749438a58eb7ecf75d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731971804903 2024-11-18T23:16:50,200 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:16:50,200 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:16:50,200 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,200 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/2165274606294e00a55d54631baf27cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3b405885b9904eaa8c0dcff36f6f7622, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f3146502761b435db3b8058276a5ed6e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f96c46d15e464347a74a5f6673efd0e1] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=146.4 K 2024-11-18T23:16:50,201 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,201 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/2165274606294e00a55d54631baf27cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3b405885b9904eaa8c0dcff36f6f7622, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f3146502761b435db3b8058276a5ed6e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f96c46d15e464347a74a5f6673efd0e1] 2024-11-18T23:16:50,201 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e758373b5e1148d68ae1428d1d3dc938, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731971806159 2024-11-18T23:16:50,201 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 2165274606294e00a55d54631baf27cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731971804369 2024-11-18T23:16:50,201 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3821fc2c4d943499e15205d57037404, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731971808500 2024-11-18T23:16:50,201 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b405885b9904eaa8c0dcff36f6f7622, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731971804903 2024-11-18T23:16:50,204 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f3146502761b435db3b8058276a5ed6e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731971806159 2024-11-18T23:16:50,205 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f96c46d15e464347a74a5f6673efd0e1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731971808500 2024-11-18T23:16:50,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-18T23:16:50,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,217 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=127.47 KB heapSize=371.88 KB 2024-11-18T23:16:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:50,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:16:50,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f852be67b8c8486a8ef514f2e1d907af_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971808644/Put/seqid=0 2024-11-18T23:16:50,249 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:50,250 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/ec422eac6e0848de99c84dfddc2b6c45 is 50, key is test_row_0/B:col10/1731971808560/Put/seqid=0 2024-11-18T23:16:50,266 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:50,274 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118ef150f6ccea44bc9b2e1236b77d773bb_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:50,277 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118ef150f6ccea44bc9b2e1236b77d773bb_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:50,278 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ef150f6ccea44bc9b2e1236b77d773bb_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:50,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742176_1352 (size=12304) 2024-11-18T23:16:50,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742177_1353 (size=12443) 2024-11-18T23:16:50,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:50,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742178_1354 (size=4469) 2024-11-18T23:16:50,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118f852be67b8c8486a8ef514f2e1d907af_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f852be67b8c8486a8ef514f2e1d907af_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/e9e0781bae2f4495af5887b82053b0e1, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:50,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/e9e0781bae2f4495af5887b82053b0e1 is 175, key is test_row_0/A:col10/1731971808644/Put/seqid=0 2024-11-18T23:16:50,331 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/ec422eac6e0848de99c84dfddc2b6c45 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ec422eac6e0848de99c84dfddc2b6c45 2024-11-18T23:16:50,338 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into ec422eac6e0848de99c84dfddc2b6c45(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:50,338 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:50,338 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=12, startTime=1731971810195; duration=0sec 2024-11-18T23:16:50,339 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:50,339 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:16:50,339 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:50,346 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:50,346 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:16:50,346 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,346 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8b26eb7a49324ea5b0b84c3decb676ff, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/5b596daa5779478d8202543dedf11701, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/9f1dd569067842d6818e844b91cc28d4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=35.5 K 2024-11-18T23:16:50,347 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b26eb7a49324ea5b0b84c3decb676ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731971804903 2024-11-18T23:16:50,347 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b596daa5779478d8202543dedf11701, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731971806159 2024-11-18T23:16:50,347 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f1dd569067842d6818e844b91cc28d4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731971808500 2024-11-18T23:16:50,362 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#302 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:50,362 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/6c0904c3f0ae472c8ad9bb6cf085710a is 50, key is test_row_0/C:col10/1731971808560/Put/seqid=0 2024-11-18T23:16:50,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742179_1355 (size=31105) 2024-11-18T23:16:50,367 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/e9e0781bae2f4495af5887b82053b0e1 2024-11-18T23:16:50,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T23:16:50,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/a030a49d9434486eba7979cea857efc1 is 50, key is test_row_0/B:col10/1731971808644/Put/seqid=0 2024-11-18T23:16:50,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742180_1356 (size=12443) 2024-11-18T23:16:50,411 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/6c0904c3f0ae472c8ad9bb6cf085710a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/6c0904c3f0ae472c8ad9bb6cf085710a 2024-11-18T23:16:50,419 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into 6c0904c3f0ae472c8ad9bb6cf085710a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:50,419 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:50,419 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=13, startTime=1731971810195; duration=0sec 2024-11-18T23:16:50,419 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:50,419 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:16:50,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742181_1357 (size=12151) 2024-11-18T23:16:50,445 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/a030a49d9434486eba7979cea857efc1 2024-11-18T23:16:50,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/7b2c2a4462464ad7b1fda9eae9b3958e is 50, key is test_row_0/C:col10/1731971808644/Put/seqid=0 2024-11-18T23:16:50,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742182_1358 (size=12151) 2024-11-18T23:16:50,731 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#301 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:50,731 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/b380c8c5b7424255b6e5f66d716ff898 is 175, key is test_row_0/A:col10/1731971808560/Put/seqid=0 2024-11-18T23:16:50,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742183_1359 (size=31397) 2024-11-18T23:16:50,771 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/b380c8c5b7424255b6e5f66d716ff898 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/b380c8c5b7424255b6e5f66d716ff898 2024-11-18T23:16:50,776 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into b380c8c5b7424255b6e5f66d716ff898(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:50,776 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:50,776 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=12, startTime=1731971810193; duration=0sec 2024-11-18T23:16:50,777 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:50,777 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:16:50,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:50,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:50,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:50,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971870862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:50,890 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 49690c5aec791d09014e5d76c85e7360 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:16:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971870878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:50,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:50,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971870885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:50,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971870887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:50,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971870892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:50,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:50,932 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/7b2c2a4462464ad7b1fda9eae9b3958e 2024-11-18T23:16:50,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/e9e0781bae2f4495af5887b82053b0e1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e9e0781bae2f4495af5887b82053b0e1 2024-11-18T23:16:50,960 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e9e0781bae2f4495af5887b82053b0e1, entries=150, sequenceid=157, filesize=30.4 K 2024-11-18T23:16:50,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/a030a49d9434486eba7979cea857efc1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a030a49d9434486eba7979cea857efc1 2024-11-18T23:16:50,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a030a49d9434486eba7979cea857efc1, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T23:16:50,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/7b2c2a4462464ad7b1fda9eae9b3958e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7b2c2a4462464ad7b1fda9eae9b3958e 2024-11-18T23:16:50,984 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7b2c2a4462464ad7b1fda9eae9b3958e, entries=150, sequenceid=157, filesize=11.9 K 2024-11-18T23:16:50,990 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~371.83 KB/380752, currentSize=46.96 KB/48090 for 49690c5aec791d09014e5d76c85e7360 in 774ms, sequenceid=157, compaction requested=false 2024-11-18T23:16:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:50,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-18T23:16:50,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-18T23:16:50,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-18T23:16:50,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7440 sec 2024-11-18T23:16:51,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.7530 sec 2024-11-18T23:16:51,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T23:16:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:51,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:51,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:51,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:51,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:51,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:51,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b616b0ff25ba42158bfe400fe485abd1_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971811022/Put/seqid=0 2024-11-18T23:16:51,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742184_1360 (size=12304) 2024-11-18T23:16:51,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971871160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971871167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971871167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971871170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971871288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971871288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971871297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971871297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-18T23:16:51,388 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:51,389 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-18T23:16:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T23:16:51,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:51,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:51,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:51,479 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:51,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b616b0ff25ba42158bfe400fe485abd1_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b616b0ff25ba42158bfe400fe485abd1_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:51,489 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4e0c3db0429d4b918459f0990a299f24, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:51,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4e0c3db0429d4b918459f0990a299f24 is 175, key is test_row_0/A:col10/1731971811022/Put/seqid=0 2024-11-18T23:16:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T23:16:51,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742185_1361 (size=31105) 2024-11-18T23:16:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971871508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971871508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971871508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971871509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,519 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4e0c3db0429d4b918459f0990a299f24 2024-11-18T23:16:51,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/ae235dbaebc64ab3a96ee316ab147754 is 50, key is test_row_0/B:col10/1731971811022/Put/seqid=0 2024-11-18T23:16:51,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-18T23:16:51,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:51,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742186_1362 (size=12151) 2024-11-18T23:16:51,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-18T23:16:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:51,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T23:16:51,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971871828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971871836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971871836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971871828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:51,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:51,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-18T23:16:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:51,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:51,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:51,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/ae235dbaebc64ab3a96ee316ab147754 2024-11-18T23:16:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T23:16:52,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-18T23:16:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:52,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/76e5d579dcc94f99a9a23cb18fdb4274 is 50, key is test_row_0/C:col10/1731971811022/Put/seqid=0 2024-11-18T23:16:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:52,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:52,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:52,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:52,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742187_1363 (size=12151) 2024-11-18T23:16:52,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/76e5d579dcc94f99a9a23cb18fdb4274 2024-11-18T23:16:52,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4e0c3db0429d4b918459f0990a299f24 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4e0c3db0429d4b918459f0990a299f24 2024-11-18T23:16:52,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4e0c3db0429d4b918459f0990a299f24, entries=150, sequenceid=172, filesize=30.4 K 2024-11-18T23:16:52,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/ae235dbaebc64ab3a96ee316ab147754 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ae235dbaebc64ab3a96ee316ab147754 2024-11-18T23:16:52,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ae235dbaebc64ab3a96ee316ab147754, entries=150, sequenceid=172, filesize=11.9 K 2024-11-18T23:16:52,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/76e5d579dcc94f99a9a23cb18fdb4274 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/76e5d579dcc94f99a9a23cb18fdb4274 2024-11-18T23:16:52,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/76e5d579dcc94f99a9a23cb18fdb4274, entries=150, sequenceid=172, filesize=11.9 K 2024-11-18T23:16:52,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 49690c5aec791d09014e5d76c85e7360 in 1111ms, sequenceid=172, compaction requested=true 2024-11-18T23:16:52,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:52,135 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:52,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:52,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:52,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:52,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:52,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:52,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:52,137 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93607 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:52,137 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:16:52,137 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:52,137 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/b380c8c5b7424255b6e5f66d716ff898, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e9e0781bae2f4495af5887b82053b0e1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4e0c3db0429d4b918459f0990a299f24] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=91.4 K 2024-11-18T23:16:52,137 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:52,137 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/b380c8c5b7424255b6e5f66d716ff898, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e9e0781bae2f4495af5887b82053b0e1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4e0c3db0429d4b918459f0990a299f24] 2024-11-18T23:16:52,139 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:52,140 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b380c8c5b7424255b6e5f66d716ff898, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731971808500 2024-11-18T23:16:52,141 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9e0781bae2f4495af5887b82053b0e1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731971808644 2024-11-18T23:16:52,145 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:52,145 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:16:52,145 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:52,145 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ec422eac6e0848de99c84dfddc2b6c45, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a030a49d9434486eba7979cea857efc1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ae235dbaebc64ab3a96ee316ab147754] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=35.9 K 2024-11-18T23:16:52,145 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e0c3db0429d4b918459f0990a299f24, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731971810864 2024-11-18T23:16:52,148 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ec422eac6e0848de99c84dfddc2b6c45, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731971808500 2024-11-18T23:16:52,149 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a030a49d9434486eba7979cea857efc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731971808644 2024-11-18T23:16:52,152 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ae235dbaebc64ab3a96ee316ab147754, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731971810864 2024-11-18T23:16:52,170 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:52,173 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#309 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:52,174 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/65980a91612b4f0f8f8283f1f364d8af is 50, key is test_row_0/B:col10/1731971811022/Put/seqid=0 2024-11-18T23:16:52,181 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118c988019ae2ee4b24a646f2867bb87254_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:52,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-18T23:16:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:52,182 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T23:16:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:52,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:52,183 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118c988019ae2ee4b24a646f2867bb87254_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:52,183 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118c988019ae2ee4b24a646f2867bb87254_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:52,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742188_1364 (size=12595) 2024-11-18T23:16:52,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181a66c23ac2724bec8301525bda6020a8_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971811165/Put/seqid=0 2024-11-18T23:16:52,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742189_1365 (size=4469) 2024-11-18T23:16:52,248 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#308 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:52,249 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/67df66a91b2a45a9a5be8b6b970cd9c5 is 175, key is test_row_0/A:col10/1731971811022/Put/seqid=0 2024-11-18T23:16:52,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742190_1366 (size=31549) 2024-11-18T23:16:52,286 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/67df66a91b2a45a9a5be8b6b970cd9c5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/67df66a91b2a45a9a5be8b6b970cd9c5 2024-11-18T23:16:52,293 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into 67df66a91b2a45a9a5be8b6b970cd9c5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:52,294 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:52,294 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=13, startTime=1731971812135; duration=0sec 2024-11-18T23:16:52,294 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:52,294 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:16:52,294 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:52,297 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:52,297 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:16:52,297 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:52,297 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/6c0904c3f0ae472c8ad9bb6cf085710a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7b2c2a4462464ad7b1fda9eae9b3958e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/76e5d579dcc94f99a9a23cb18fdb4274] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=35.9 K 2024-11-18T23:16:52,298 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c0904c3f0ae472c8ad9bb6cf085710a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731971808500 2024-11-18T23:16:52,299 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b2c2a4462464ad7b1fda9eae9b3958e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731971808644 2024-11-18T23:16:52,299 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e5d579dcc94f99a9a23cb18fdb4274, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731971810864 2024-11-18T23:16:52,323 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:52,324 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/1989861927ea495da6db0866fb4a64bb is 50, key is test_row_0/C:col10/1731971811022/Put/seqid=0 2024-11-18T23:16:52,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742191_1367 (size=12304) 2024-11-18T23:16:52,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:52,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742192_1368 (size=12595) 2024-11-18T23:16:52,350 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181a66c23ac2724bec8301525bda6020a8_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a66c23ac2724bec8301525bda6020a8_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:52,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cfc97859b31f4d69b63f39523d23d489, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:52,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cfc97859b31f4d69b63f39523d23d489 is 175, key is test_row_0/A:col10/1731971811165/Put/seqid=0 2024-11-18T23:16:52,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:52,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742193_1369 (size=31105) 2024-11-18T23:16:52,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971872388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971872390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971872391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971872400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971872520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971872520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971872523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T23:16:52,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971872523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,616 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/65980a91612b4f0f8f8283f1f364d8af as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/65980a91612b4f0f8f8283f1f364d8af 2024-11-18T23:16:52,629 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into 65980a91612b4f0f8f8283f1f364d8af(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:52,629 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:52,629 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=13, startTime=1731971812136; duration=0sec 2024-11-18T23:16:52,629 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:52,629 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:16:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971872737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971872738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971872748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971872748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:52,756 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/1989861927ea495da6db0866fb4a64bb as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1989861927ea495da6db0866fb4a64bb 2024-11-18T23:16:52,769 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into 1989861927ea495da6db0866fb4a64bb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:52,769 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:52,769 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=13, startTime=1731971812136; duration=0sec 2024-11-18T23:16:52,770 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:52,770 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:16:52,780 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cfc97859b31f4d69b63f39523d23d489 2024-11-18T23:16:52,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/4e1fd478929d4ef6b30224f3d305e48a is 50, key is test_row_0/B:col10/1731971811165/Put/seqid=0 2024-11-18T23:16:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742194_1370 (size=12151) 2024-11-18T23:16:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971872917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:52,932 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 49690c5aec791d09014e5d76c85e7360 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:16:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:52,933 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971873097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971873098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971873098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971873103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,239 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/4e1fd478929d4ef6b30224f3d305e48a 2024-11-18T23:16:53,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/08247cd4706644b38b1d86817e782b3d is 50, key is test_row_0/C:col10/1731971811165/Put/seqid=0 2024-11-18T23:16:53,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742195_1371 (size=12151) 2024-11-18T23:16:53,291 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/08247cd4706644b38b1d86817e782b3d 2024-11-18T23:16:53,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/cfc97859b31f4d69b63f39523d23d489 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cfc97859b31f4d69b63f39523d23d489 2024-11-18T23:16:53,306 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cfc97859b31f4d69b63f39523d23d489, entries=150, sequenceid=192, filesize=30.4 K 2024-11-18T23:16:53,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/4e1fd478929d4ef6b30224f3d305e48a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4e1fd478929d4ef6b30224f3d305e48a 2024-11-18T23:16:53,313 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4e1fd478929d4ef6b30224f3d305e48a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-18T23:16:53,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/08247cd4706644b38b1d86817e782b3d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/08247cd4706644b38b1d86817e782b3d 2024-11-18T23:16:53,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/08247cd4706644b38b1d86817e782b3d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-18T23:16:53,338 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 49690c5aec791d09014e5d76c85e7360 in 1156ms, sequenceid=192, compaction requested=false 2024-11-18T23:16:53,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:53,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-18T23:16:53,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-18T23:16:53,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-18T23:16:53,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9460 sec 2024-11-18T23:16:53,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.9560 sec 2024-11-18T23:16:53,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-18T23:16:53,539 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:53,539 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:53,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-18T23:16:53,541 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:53,542 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:53,542 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T23:16:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:53,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:16:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:53,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:53,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118096799882a994d6588aef753b5c4cd55_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971812389/Put/seqid=0 2024-11-18T23:16:53,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742196_1372 (size=14794) 2024-11-18T23:16:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T23:16:53,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971873653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971873659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971873662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,675 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971873663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:53,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:53,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971873777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971873777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971873779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971873791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:53,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:53,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:53,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:53,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:53,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:53,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:53,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:53,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T23:16:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971873998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:54,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:54,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971873998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971874000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971874019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,040 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:54,050 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118096799882a994d6588aef753b5c4cd55_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118096799882a994d6588aef753b5c4cd55_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:54,053 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a914f5e9521c4ad68af7d68598508b2b, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:54,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a914f5e9521c4ad68af7d68598508b2b is 175, key is test_row_0/A:col10/1731971812389/Put/seqid=0 2024-11-18T23:16:54,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742197_1373 (size=39749) 2024-11-18T23:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T23:16:54,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:54,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:54,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:54,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971874328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971874328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971874328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971874337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:54,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:54,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a914f5e9521c4ad68af7d68598508b2b 2024-11-18T23:16:54,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/da395d69561a45c5a10be193ea11f664 is 50, key is test_row_0/B:col10/1731971812389/Put/seqid=0 2024-11-18T23:16:54,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742198_1374 (size=12151) 2024-11-18T23:16:54,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T23:16:54,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:54,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:54,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971874848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971874850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971874850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971874852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:54,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/da395d69561a45c5a10be193ea11f664 2024-11-18T23:16:54,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:54,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:54,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/c5c14b3365b64bc5a84bb848dad3eca8 is 50, key is test_row_0/C:col10/1731971812389/Put/seqid=0 2024-11-18T23:16:55,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742199_1375 (size=12151) 2024-11-18T23:16:55,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:55,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:55,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/c5c14b3365b64bc5a84bb848dad3eca8 2024-11-18T23:16:55,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:55,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:55,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:55,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/a914f5e9521c4ad68af7d68598508b2b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a914f5e9521c4ad68af7d68598508b2b 2024-11-18T23:16:55,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a914f5e9521c4ad68af7d68598508b2b, entries=200, sequenceid=208, filesize=38.8 K 2024-11-18T23:16:55,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/da395d69561a45c5a10be193ea11f664 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/da395d69561a45c5a10be193ea11f664 2024-11-18T23:16:55,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/da395d69561a45c5a10be193ea11f664, entries=150, sequenceid=208, filesize=11.9 K 2024-11-18T23:16:55,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/c5c14b3365b64bc5a84bb848dad3eca8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/c5c14b3365b64bc5a84bb848dad3eca8 2024-11-18T23:16:55,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/c5c14b3365b64bc5a84bb848dad3eca8, entries=150, sequenceid=208, filesize=11.9 K 2024-11-18T23:16:55,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 49690c5aec791d09014e5d76c85e7360 in 1893ms, sequenceid=208, compaction requested=true 2024-11-18T23:16:55,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:55,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:55,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:55,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:55,506 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:55,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:55,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:55,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:16:55,507 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:55,507 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:55,507 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:55,508 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:16:55,508 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:16:55,508 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,508 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,508 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/67df66a91b2a45a9a5be8b6b970cd9c5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cfc97859b31f4d69b63f39523d23d489, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a914f5e9521c4ad68af7d68598508b2b] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=100.0 K 2024-11-18T23:16:55,508 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/65980a91612b4f0f8f8283f1f364d8af, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4e1fd478929d4ef6b30224f3d305e48a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/da395d69561a45c5a10be193ea11f664] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.0 K 2024-11-18T23:16:55,508 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,508 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/67df66a91b2a45a9a5be8b6b970cd9c5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cfc97859b31f4d69b63f39523d23d489, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a914f5e9521c4ad68af7d68598508b2b] 2024-11-18T23:16:55,508 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65980a91612b4f0f8f8283f1f364d8af, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731971810864 2024-11-18T23:16:55,509 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 67df66a91b2a45a9a5be8b6b970cd9c5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731971810864 2024-11-18T23:16:55,509 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting cfc97859b31f4d69b63f39523d23d489, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731971811143 2024-11-18T23:16:55,509 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a914f5e9521c4ad68af7d68598508b2b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731971812388 2024-11-18T23:16:55,509 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e1fd478929d4ef6b30224f3d305e48a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731971811143 2024-11-18T23:16:55,510 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting da395d69561a45c5a10be193ea11f664, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731971812389 2024-11-18T23:16:55,537 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:55,557 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:55,558 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/a6591bddaec34aa786d3cfaccc523ce4 is 50, key is test_row_0/B:col10/1731971812389/Put/seqid=0 2024-11-18T23:16:55,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-18T23:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,571 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T23:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:55,571 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411180697c3575c6f4fb48426c8b6a480fc4f_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:55,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:55,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:55,574 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411180697c3575c6f4fb48426c8b6a480fc4f_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:55,574 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180697c3575c6f4fb48426c8b6a480fc4f_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:55,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742200_1376 (size=12697) 2024-11-18T23:16:55,648 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/a6591bddaec34aa786d3cfaccc523ce4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a6591bddaec34aa786d3cfaccc523ce4 2024-11-18T23:16:55,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111807e9b0ea1a94415284c98b93c818876f_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971813658/Put/seqid=0 2024-11-18T23:16:55,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742201_1377 (size=4469) 2024-11-18T23:16:55,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742202_1378 (size=12304) 2024-11-18T23:16:55,675 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#317 average throughput is 0.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:55,675 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/e7518015aaf548b3a08b317e3846a950 is 175, key is test_row_0/A:col10/1731971812389/Put/seqid=0 2024-11-18T23:16:55,676 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into a6591bddaec34aa786d3cfaccc523ce4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:55,676 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:55,677 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=13, startTime=1731971815506; duration=0sec 2024-11-18T23:16:55,677 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:55,677 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:16:55,677 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:55,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T23:16:55,692 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:55,692 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:16:55,692 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:55,692 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1989861927ea495da6db0866fb4a64bb, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/08247cd4706644b38b1d86817e782b3d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/c5c14b3365b64bc5a84bb848dad3eca8] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.0 K 2024-11-18T23:16:55,693 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1989861927ea495da6db0866fb4a64bb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731971810864 2024-11-18T23:16:55,693 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08247cd4706644b38b1d86817e782b3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731971811143 2024-11-18T23:16:55,698 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5c14b3365b64bc5a84bb848dad3eca8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731971812389 2024-11-18T23:16:55,707 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111807e9b0ea1a94415284c98b93c818876f_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807e9b0ea1a94415284c98b93c818876f_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:55,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/04f3141538144aaaa5558860bb40f1b3, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:55,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/04f3141538144aaaa5558860bb40f1b3 is 175, key is test_row_0/A:col10/1731971813658/Put/seqid=0 2024-11-18T23:16:55,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742203_1379 (size=31651) 2024-11-18T23:16:55,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742204_1380 (size=31105) 2024-11-18T23:16:55,773 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/04f3141538144aaaa5558860bb40f1b3 2024-11-18T23:16:55,778 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:55,778 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d8250a68c49b428a8b609426f41d2837 is 50, key is test_row_0/C:col10/1731971812389/Put/seqid=0 2024-11-18T23:16:55,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742205_1381 (size=12697) 2024-11-18T23:16:55,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/57c4b390242e4fd89e0ef61764fd56fa is 50, key is test_row_0/B:col10/1731971813658/Put/seqid=0 2024-11-18T23:16:55,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742206_1382 (size=12151) 2024-11-18T23:16:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:55,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:55,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971875925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:55,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971875945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971875949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971875953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:55,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971876060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971876082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971876089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971876089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,168 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/e7518015aaf548b3a08b317e3846a950 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e7518015aaf548b3a08b317e3846a950 2024-11-18T23:16:56,188 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into e7518015aaf548b3a08b317e3846a950(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:56,188 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:56,189 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=13, startTime=1731971815506; duration=0sec 2024-11-18T23:16:56,189 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:56,189 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:16:56,229 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/d8250a68c49b428a8b609426f41d2837 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d8250a68c49b428a8b609426f41d2837 2024-11-18T23:16:56,253 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into d8250a68c49b428a8b609426f41d2837(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:56,253 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:56,254 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=13, startTime=1731971815507; duration=0sec 2024-11-18T23:16:56,254 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:56,254 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:16:56,276 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/57c4b390242e4fd89e0ef61764fd56fa 2024-11-18T23:16:56,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/8f12dd45116149f7a07a41f1f0f52668 is 50, key is test_row_0/C:col10/1731971813658/Put/seqid=0 2024-11-18T23:16:56,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971876288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971876297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742207_1383 (size=12151) 2024-11-18T23:16:56,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971876318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,322 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971876332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971876601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971876620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,634 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971876628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,640 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971876647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:56,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:56,716 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/8f12dd45116149f7a07a41f1f0f52668 2024-11-18T23:16:56,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/04f3141538144aaaa5558860bb40f1b3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/04f3141538144aaaa5558860bb40f1b3 2024-11-18T23:16:56,774 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/04f3141538144aaaa5558860bb40f1b3, entries=150, sequenceid=227, filesize=30.4 K 2024-11-18T23:16:56,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/57c4b390242e4fd89e0ef61764fd56fa as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/57c4b390242e4fd89e0ef61764fd56fa 2024-11-18T23:16:56,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/57c4b390242e4fd89e0ef61764fd56fa, entries=150, sequenceid=227, filesize=11.9 K 2024-11-18T23:16:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/8f12dd45116149f7a07a41f1f0f52668 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8f12dd45116149f7a07a41f1f0f52668 2024-11-18T23:16:56,847 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8f12dd45116149f7a07a41f1f0f52668, entries=150, sequenceid=227, filesize=11.9 K 2024-11-18T23:16:56,854 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 49690c5aec791d09014e5d76c85e7360 in 1283ms, sequenceid=227, compaction requested=false 2024-11-18T23:16:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-18T23:16:56,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-18T23:16:56,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-18T23:16:56,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3260 sec 2024-11-18T23:16:56,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 3.3400 sec 2024-11-18T23:16:56,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-18T23:16:56,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:56,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:56,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:56,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118285772b3b64b4875b813b3fef8162561_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971815948/Put/seqid=0 2024-11-18T23:16:57,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742208_1384 (size=14794) 2024-11-18T23:16:57,058 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:57,066 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118285772b3b64b4875b813b3fef8162561_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118285772b3b64b4875b813b3fef8162561_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:57,067 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/6396e5e4aec042449cfe4b26dc12ebc5, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:57,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/6396e5e4aec042449cfe4b26dc12ebc5 is 175, key is test_row_0/A:col10/1731971815948/Put/seqid=0 2024-11-18T23:16:57,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742209_1385 (size=39749) 2024-11-18T23:16:57,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971877147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971877149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971877166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971877167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,186 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971877172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971877268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971877297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971877297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971877487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,516 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/6396e5e4aec042449cfe4b26dc12ebc5 2024-11-18T23:16:57,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971877517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971877517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/79a06a8365ad4eb88057dc808db3f51f is 50, key is test_row_0/B:col10/1731971815948/Put/seqid=0 2024-11-18T23:16:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742210_1386 (size=12151) 2024-11-18T23:16:57,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-18T23:16:57,700 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:57,700 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-18T23:16:57,703 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T23:16:57,704 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:57,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T23:16:57,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971877807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971877828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971877828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:57,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:57,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T23:16:57,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:57,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:57,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:57,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:57,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/79a06a8365ad4eb88057dc808db3f51f 2024-11-18T23:16:58,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T23:16:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:58,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:58,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T23:16:58,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/2d50b9355f8e4a9b8a331a176af897f2 is 50, key is test_row_0/C:col10/1731971815948/Put/seqid=0 2024-11-18T23:16:58,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742211_1387 (size=12151) 2024-11-18T23:16:58,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/2d50b9355f8e4a9b8a331a176af897f2 2024-11-18T23:16:58,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/6396e5e4aec042449cfe4b26dc12ebc5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/6396e5e4aec042449cfe4b26dc12ebc5 2024-11-18T23:16:58,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T23:16:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:58,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:16:58,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/6396e5e4aec042449cfe4b26dc12ebc5, entries=200, sequenceid=245, filesize=38.8 K 2024-11-18T23:16:58,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971878183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,191 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in 49690c5aec791d09014e5d76c85e7360 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:16:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,191 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/79a06a8365ad4eb88057dc808db3f51f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/79a06a8365ad4eb88057dc808db3f51f 2024-11-18T23:16:58,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971878198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,209 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in 49690c5aec791d09014e5d76c85e7360 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:16:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/79a06a8365ad4eb88057dc808db3f51f, entries=150, sequenceid=245, filesize=11.9 K 2024-11-18T23:16:58,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/2d50b9355f8e4a9b8a331a176af897f2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2d50b9355f8e4a9b8a331a176af897f2 2024-11-18T23:16:58,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2d50b9355f8e4a9b8a331a176af897f2, entries=150, sequenceid=245, filesize=11.9 K 2024-11-18T23:16:58,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 49690c5aec791d09014e5d76c85e7360 in 1291ms, sequenceid=245, compaction requested=true 2024-11-18T23:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:58,241 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:16:58,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:58,241 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:58,252 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:58,252 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:16:58,252 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,252 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a6591bddaec34aa786d3cfaccc523ce4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/57c4b390242e4fd89e0ef61764fd56fa, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/79a06a8365ad4eb88057dc808db3f51f] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.1 K 2024-11-18T23:16:58,257 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:58,257 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a6591bddaec34aa786d3cfaccc523ce4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731971812389 2024-11-18T23:16:58,257 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:16:58,257 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,257 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e7518015aaf548b3a08b317e3846a950, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/04f3141538144aaaa5558860bb40f1b3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/6396e5e4aec042449cfe4b26dc12ebc5] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=100.1 K 2024-11-18T23:16:58,257 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,257 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e7518015aaf548b3a08b317e3846a950, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/04f3141538144aaaa5558860bb40f1b3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/6396e5e4aec042449cfe4b26dc12ebc5] 2024-11-18T23:16:58,262 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c4b390242e4fd89e0ef61764fd56fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731971813658 2024-11-18T23:16:58,263 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7518015aaf548b3a08b317e3846a950, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731971812389 2024-11-18T23:16:58,263 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a06a8365ad4eb88057dc808db3f51f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731971815943 2024-11-18T23:16:58,265 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f3141538144aaaa5558860bb40f1b3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731971813658 2024-11-18T23:16:58,267 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6396e5e4aec042449cfe4b26dc12ebc5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731971815909 2024-11-18T23:16:58,296 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:58,297 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/82f55d28c5ce4f43a983196360d624e2 is 50, key is test_row_0/B:col10/1731971815948/Put/seqid=0 2024-11-18T23:16:58,297 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:58,301 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411180a23d714a561422490e8f0e35e13c45d_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:58,303 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411180a23d714a561422490e8f0e35e13c45d_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:58,303 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180a23d714a561422490e8f0e35e13c45d_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T23:16:58,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-18T23:16:58,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T23:16:58,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:58,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:58,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:58,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742212_1388 (size=12799) 2024-11-18T23:16:58,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:58,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:16:58,360 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/82f55d28c5ce4f43a983196360d624e2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/82f55d28c5ce4f43a983196360d624e2 2024-11-18T23:16:58,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742213_1389 (size=4469) 2024-11-18T23:16:58,382 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into 82f55d28c5ce4f43a983196360d624e2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:58,382 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:58,382 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=13, startTime=1731971818241; duration=0sec 2024-11-18T23:16:58,382 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:16:58,382 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:16:58,382 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:16:58,383 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:16:58,384 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:16:58,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:58,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d8250a68c49b428a8b609426f41d2837, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8f12dd45116149f7a07a41f1f0f52668, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2d50b9355f8e4a9b8a331a176af897f2] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.1 K 2024-11-18T23:16:58,385 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d8250a68c49b428a8b609426f41d2837, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731971812389 2024-11-18T23:16:58,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118807220dbfb7e489e8a7311f2ed42749f_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971817125/Put/seqid=0 2024-11-18T23:16:58,388 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f12dd45116149f7a07a41f1f0f52668, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731971813658 2024-11-18T23:16:58,388 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d50b9355f8e4a9b8a331a176af897f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731971815943 2024-11-18T23:16:58,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742214_1390 (size=12454) 2024-11-18T23:16:58,411 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:58,411 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a49a235431504e2598aa837a2085ec03 is 50, key is test_row_0/C:col10/1731971815948/Put/seqid=0 2024-11-18T23:16:58,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971878425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,432 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742215_1391 (size=12799) 2024-11-18T23:16:58,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971878432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971878434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,479 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a49a235431504e2598aa837a2085ec03 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a49a235431504e2598aa837a2085ec03 2024-11-18T23:16:58,486 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into a49a235431504e2598aa837a2085ec03(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:58,486 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:58,486 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=13, startTime=1731971818241; duration=0sec 2024-11-18T23:16:58,487 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:58,487 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:16:58,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971878537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,548 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971878561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971878561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971878760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,776 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#327 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:16:58,777 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/98c29ad0c5604a8dbf05b07ae34e490b is 175, key is test_row_0/A:col10/1731971815948/Put/seqid=0 2024-11-18T23:16:58,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971878778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971878778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:58,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:58,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742216_1392 (size=31753) 2024-11-18T23:16:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T23:16:58,850 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118807220dbfb7e489e8a7311f2ed42749f_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118807220dbfb7e489e8a7311f2ed42749f_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:58,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4d4318446985445c8ceb0891ef7846e0, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:58,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4d4318446985445c8ceb0891ef7846e0 is 175, key is test_row_0/A:col10/1731971817125/Put/seqid=0 2024-11-18T23:16:58,868 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/98c29ad0c5604a8dbf05b07ae34e490b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/98c29ad0c5604a8dbf05b07ae34e490b 2024-11-18T23:16:58,881 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into 98c29ad0c5604a8dbf05b07ae34e490b(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:16:58,881 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:58,881 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=13, startTime=1731971818241; duration=0sec 2024-11-18T23:16:58,881 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:16:58,881 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:16:58,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742217_1393 (size=31255) 2024-11-18T23:16:59,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971879078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971879100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971879100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,293 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4d4318446985445c8ceb0891ef7846e0 2024-11-18T23:16:59,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/9e9a18b28d304d568fdc517b1ed397ba is 50, key is test_row_0/B:col10/1731971817125/Put/seqid=0 2024-11-18T23:16:59,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742218_1394 (size=12301) 2024-11-18T23:16:59,353 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/9e9a18b28d304d568fdc517b1ed397ba 2024-11-18T23:16:59,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/60745606a7f7428a8374a7caf613ea67 is 50, key is test_row_0/C:col10/1731971817125/Put/seqid=0 2024-11-18T23:16:59,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742219_1395 (size=12301) 2024-11-18T23:16:59,403 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/60745606a7f7428a8374a7caf613ea67 2024-11-18T23:16:59,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/4d4318446985445c8ceb0891ef7846e0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4d4318446985445c8ceb0891ef7846e0 2024-11-18T23:16:59,419 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4d4318446985445c8ceb0891ef7846e0, entries=150, sequenceid=262, filesize=30.5 K 2024-11-18T23:16:59,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/9e9a18b28d304d568fdc517b1ed397ba as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9e9a18b28d304d568fdc517b1ed397ba 2024-11-18T23:16:59,436 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9e9a18b28d304d568fdc517b1ed397ba, entries=150, sequenceid=262, filesize=12.0 K 2024-11-18T23:16:59,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/60745606a7f7428a8374a7caf613ea67 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/60745606a7f7428a8374a7caf613ea67 2024-11-18T23:16:59,462 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/60745606a7f7428a8374a7caf613ea67, entries=150, sequenceid=262, filesize=12.0 K 2024-11-18T23:16:59,463 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 49690c5aec791d09014e5d76c85e7360 in 1133ms, sequenceid=262, compaction requested=false 2024-11-18T23:16:59,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:16:59,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:16:59,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-18T23:16:59,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-18T23:16:59,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-18T23:16:59,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7600 sec 2024-11-18T23:16:59,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.7680 sec 2024-11-18T23:16:59,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:16:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:16:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:16:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:16:59,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:16:59,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:59,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118570c548b69b5490d975a28c90d1e8f9f_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971819598/Put/seqid=0 2024-11-18T23:16:59,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742220_1396 (size=14994) 2024-11-18T23:16:59,661 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:16:59,679 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118570c548b69b5490d975a28c90d1e8f9f_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118570c548b69b5490d975a28c90d1e8f9f_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:16:59,680 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/d2c74fac3f284d9c97cbe73c84987da3, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:16:59,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/d2c74fac3f284d9c97cbe73c84987da3 is 175, key is test_row_0/A:col10/1731971819598/Put/seqid=0 2024-11-18T23:16:59,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742221_1397 (size=39949) 2024-11-18T23:16:59,719 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/d2c74fac3f284d9c97cbe73c84987da3 2024-11-18T23:16:59,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/91b114ad96634ad3a05b6dcad3a03033 is 50, key is test_row_0/B:col10/1731971819598/Put/seqid=0 2024-11-18T23:16:59,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971879708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971879733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971879733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742222_1398 (size=12301) 2024-11-18T23:16:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-18T23:16:59,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971879842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,858 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:16:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971879858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:16:59,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971879859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:16:59,861 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:16:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:16:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:16:59,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-18T23:16:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T23:16:59,873 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:16:59,877 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:16:59,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:16:59,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T23:17:00,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T23:17:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:17:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971880067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971880068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971880068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,085 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/91b114ad96634ad3a05b6dcad3a03033 2024-11-18T23:17:00,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T23:17:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:17:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T23:17:00,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/db1bf058369f4dc2b82ec71d0dac64f3 is 50, key is test_row_0/C:col10/1731971819598/Put/seqid=0 2024-11-18T23:17:00,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:59004 deadline: 1731971880210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,219 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in 49690c5aec791d09014e5d76c85e7360 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742223_1399 (size=12301) 2024-11-18T23:17:00,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58962 deadline: 1731971880229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,240 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in 49690c5aec791d09014e5d76c85e7360 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T23:17:00,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:17:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971880388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971880389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,391 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971880397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T23:17:00,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T23:17:00,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:17:00,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/db1bf058369f4dc2b82ec71d0dac64f3 2024-11-18T23:17:00,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/d2c74fac3f284d9c97cbe73c84987da3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/d2c74fac3f284d9c97cbe73c84987da3 2024-11-18T23:17:00,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/d2c74fac3f284d9c97cbe73c84987da3, entries=200, sequenceid=281, filesize=39.0 K 2024-11-18T23:17:00,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/91b114ad96634ad3a05b6dcad3a03033 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/91b114ad96634ad3a05b6dcad3a03033 2024-11-18T23:17:00,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T23:17:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:17:00,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:00,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/91b114ad96634ad3a05b6dcad3a03033, entries=150, sequenceid=281, filesize=12.0 K 2024-11-18T23:17:00,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/db1bf058369f4dc2b82ec71d0dac64f3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/db1bf058369f4dc2b82ec71d0dac64f3 2024-11-18T23:17:00,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/db1bf058369f4dc2b82ec71d0dac64f3, entries=150, sequenceid=281, filesize=12.0 K 2024-11-18T23:17:00,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 49690c5aec791d09014e5d76c85e7360 in 1104ms, sequenceid=281, compaction requested=true 2024-11-18T23:17:00,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:00,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:00,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:00,716 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:00,716 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:00,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:00,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:00,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:00,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:00,718 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:00,718 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:17:00,719 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,719 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/98c29ad0c5604a8dbf05b07ae34e490b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4d4318446985445c8ceb0891ef7846e0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/d2c74fac3f284d9c97cbe73c84987da3] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=100.5 K 2024-11-18T23:17:00,719 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,719 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/98c29ad0c5604a8dbf05b07ae34e490b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4d4318446985445c8ceb0891ef7846e0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/d2c74fac3f284d9c97cbe73c84987da3] 2024-11-18T23:17:00,720 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:00,720 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:17:00,720 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,720 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/82f55d28c5ce4f43a983196360d624e2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9e9a18b28d304d568fdc517b1ed397ba, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/91b114ad96634ad3a05b6dcad3a03033] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.5 K 2024-11-18T23:17:00,721 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f55d28c5ce4f43a983196360d624e2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731971815943 2024-11-18T23:17:00,721 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9a18b28d304d568fdc517b1ed397ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731971817125 2024-11-18T23:17:00,721 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c29ad0c5604a8dbf05b07ae34e490b, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731971815943 2024-11-18T23:17:00,727 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d4318446985445c8ceb0891ef7846e0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731971817125 2024-11-18T23:17:00,727 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2c74fac3f284d9c97cbe73c84987da3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731971818385 2024-11-18T23:17:00,728 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 91b114ad96634ad3a05b6dcad3a03033, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731971818385 2024-11-18T23:17:00,759 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:00,760 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/4851102f6f644c2387fbba9b0d9180e7 is 50, key is test_row_0/B:col10/1731971819598/Put/seqid=0 2024-11-18T23:17:00,763 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:00,766 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411185c05c48cf1b84279bf94344834e63cbb_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:00,772 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411185c05c48cf1b84279bf94344834e63cbb_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:00,772 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185c05c48cf1b84279bf94344834e63cbb_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:00,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742224_1400 (size=13051) 2024-11-18T23:17:00,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742225_1401 (size=4469) 2024-11-18T23:17:00,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-18T23:17:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:00,827 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:17:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:17:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:17:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:17:00,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111891d3a7d22026434884b91ce924f7c012_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971819722/Put/seqid=0 2024-11-18T23:17:00,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. as already flushing 2024-11-18T23:17:00,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:00,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742226_1402 (size=12454) 2024-11-18T23:17:01,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T23:17:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971881033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971881037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971881061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971881169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971881169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,189 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971881199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,216 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/4851102f6f644c2387fbba9b0d9180e7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4851102f6f644c2387fbba9b0d9180e7 2024-11-18T23:17:01,228 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#336 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:01,229 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/eb995c43b6a742089387d55661ca8d0b is 175, key is test_row_0/A:col10/1731971819598/Put/seqid=0 2024-11-18T23:17:01,244 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into 4851102f6f644c2387fbba9b0d9180e7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:01,244 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:01,244 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=13, startTime=1731971820716; duration=0sec 2024-11-18T23:17:01,245 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:01,245 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:17:01,245 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:01,250 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:01,250 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:17:01,250 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:01,250 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a49a235431504e2598aa837a2085ec03, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/60745606a7f7428a8374a7caf613ea67, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/db1bf058369f4dc2b82ec71d0dac64f3] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.5 K 2024-11-18T23:17:01,252 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a49a235431504e2598aa837a2085ec03, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731971815943 2024-11-18T23:17:01,252 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 60745606a7f7428a8374a7caf613ea67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731971817125 2024-11-18T23:17:01,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742227_1403 (size=32005) 2024-11-18T23:17:01,259 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting db1bf058369f4dc2b82ec71d0dac64f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731971818385 2024-11-18T23:17:01,295 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#338 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:01,295 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/17fade7538cb4cd4843d65c084af9a4f is 50, key is test_row_0/C:col10/1731971819598/Put/seqid=0 2024-11-18T23:17:01,296 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/eb995c43b6a742089387d55661ca8d0b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eb995c43b6a742089387d55661ca8d0b 2024-11-18T23:17:01,329 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into eb995c43b6a742089387d55661ca8d0b(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:01,329 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:01,329 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=13, startTime=1731971820716; duration=0sec 2024-11-18T23:17:01,329 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:01,329 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:17:01,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742228_1404 (size=13051) 2024-11-18T23:17:01,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:01,368 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111891d3a7d22026434884b91ce924f7c012_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891d3a7d22026434884b91ce924f7c012_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:01,372 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/17fade7538cb4cd4843d65c084af9a4f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/17fade7538cb4cd4843d65c084af9a4f 2024-11-18T23:17:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/dbe87036bb2647ba9fa4cddffbe3d5f7, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/dbe87036bb2647ba9fa4cddffbe3d5f7 is 175, key is test_row_0/A:col10/1731971819722/Put/seqid=0 2024-11-18T23:17:01,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971881388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742229_1405 (size=31255) 2024-11-18T23:17:01,414 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into 17fade7538cb4cd4843d65c084af9a4f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:01,414 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:01,414 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=13, startTime=1731971820717; duration=0sec 2024-11-18T23:17:01,414 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:01,414 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:17:01,414 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=297, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/dbe87036bb2647ba9fa4cddffbe3d5f7 2024-11-18T23:17:01,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971881401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971881417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/3235410af06e42b39219b925636fdf16 is 50, key is test_row_0/B:col10/1731971819722/Put/seqid=0 2024-11-18T23:17:01,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742230_1406 (size=12301) 2024-11-18T23:17:01,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971881712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971881749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971881749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:01,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:01,755 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:01,767 INFO [Thread-1448 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1448. 2024-11-18T23:17:01,767 DEBUG [Thread-1448 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:01,767 DEBUG [Thread-1448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:01,771 INFO [Thread-1450 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1450. 2024-11-18T23:17:01,771 DEBUG [Thread-1450 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:01,771 DEBUG [Thread-1450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:01,772 DEBUG [Thread-1450 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,772 DEBUG [Thread-1448 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,776 INFO [Thread-1444 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1444. 2024-11-18T23:17:01,776 DEBUG [Thread-1444 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:01,776 DEBUG [Thread-1444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,777 DEBUG [Thread-1444 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,778 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:01,783 INFO [Thread-1452 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1452. 2024-11-18T23:17:01,783 DEBUG [Thread-1452 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:01,783 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,783 DEBUG [Thread-1452 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:01,790 INFO [Thread-1446 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1446. 2024-11-18T23:17:01,790 DEBUG [Thread-1446 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:01,790 DEBUG [Thread-1446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,791 DEBUG [Thread-1446 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:01,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:01,895 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/3235410af06e42b39219b925636fdf16 2024-11-18T23:17:01,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/2b1f2225f07e4582b46c896795fdfa43 is 50, key is test_row_0/C:col10/1731971819722/Put/seqid=0 2024-11-18T23:17:01,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742231_1407 (size=12301) 2024-11-18T23:17:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T23:17:02,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58922 deadline: 1731971882248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:02,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:02,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58988 deadline: 1731971882257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:02,259 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:02,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58974 deadline: 1731971882267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:02,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:02,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/2b1f2225f07e4582b46c896795fdfa43 2024-11-18T23:17:02,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/dbe87036bb2647ba9fa4cddffbe3d5f7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/dbe87036bb2647ba9fa4cddffbe3d5f7 2024-11-18T23:17:02,335 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/dbe87036bb2647ba9fa4cddffbe3d5f7, entries=150, sequenceid=297, filesize=30.5 K 2024-11-18T23:17:02,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/3235410af06e42b39219b925636fdf16 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/3235410af06e42b39219b925636fdf16 2024-11-18T23:17:02,341 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/3235410af06e42b39219b925636fdf16, entries=150, sequenceid=297, filesize=12.0 K 2024-11-18T23:17:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/2b1f2225f07e4582b46c896795fdfa43 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2b1f2225f07e4582b46c896795fdfa43 2024-11-18T23:17:02,348 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2b1f2225f07e4582b46c896795fdfa43, entries=150, sequenceid=297, filesize=12.0 K 2024-11-18T23:17:02,349 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 49690c5aec791d09014e5d76c85e7360 in 1522ms, sequenceid=297, compaction requested=false 2024-11-18T23:17:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:02,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-18T23:17:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-18T23:17:02,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-18T23:17:02,353 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4740 sec 2024-11-18T23:17:02,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.4920 sec 2024-11-18T23:17:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:03,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-18T23:17:03,274 INFO [Thread-1433 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1433. 2024-11-18T23:17:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:17:03,274 DEBUG [Thread-1433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:03,274 DEBUG [Thread-1433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:17:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:17:03,274 DEBUG [Thread-1433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:03,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:03,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:03,275 INFO [Thread-1439 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1439. 2024-11-18T23:17:03,275 DEBUG [Thread-1439 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:03,275 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:03,275 DEBUG [Thread-1439 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:03,275 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:03,279 INFO [Thread-1435 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1435. 2024-11-18T23:17:03,280 DEBUG [Thread-1435 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:03,280 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:03,280 DEBUG [Thread-1435 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:03,280 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:03,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189be369aa3ccf40ffb5f9575be71398c6_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_0/A:col10/1731971823268/Put/seqid=0 2024-11-18T23:17:03,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742232_1408 (size=12454) 2024-11-18T23:17:03,323 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:03,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411189be369aa3ccf40ffb5f9575be71398c6_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189be369aa3ccf40ffb5f9575be71398c6_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:03,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/91a906231eee439a88a39251a8aaa4c0, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:03,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/91a906231eee439a88a39251a8aaa4c0 is 175, key is test_row_0/A:col10/1731971823268/Put/seqid=0 2024-11-18T23:17:03,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742233_1409 (size=31255) 2024-11-18T23:17:03,768 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/91a906231eee439a88a39251a8aaa4c0 2024-11-18T23:17:03,776 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/199dee39b8af4562946a435f050666b4 is 50, key is test_row_0/B:col10/1731971823268/Put/seqid=0 2024-11-18T23:17:03,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742234_1410 (size=12301) 2024-11-18T23:17:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-18T23:17:04,037 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:04,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/199dee39b8af4562946a435f050666b4 2024-11-18T23:17:04,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/1a7a87fff34942198ba5eca19b79df26 is 50, key is test_row_0/C:col10/1731971823268/Put/seqid=0 2024-11-18T23:17:04,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742235_1411 (size=12301) 2024-11-18T23:17:04,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/1a7a87fff34942198ba5eca19b79df26 2024-11-18T23:17:04,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/91a906231eee439a88a39251a8aaa4c0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/91a906231eee439a88a39251a8aaa4c0 2024-11-18T23:17:04,259 INFO [Thread-1437 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1437. 2024-11-18T23:17:04,260 DEBUG [Thread-1437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:04,260 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:04,260 DEBUG [Thread-1437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:04,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:04,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/91a906231eee439a88a39251a8aaa4c0, entries=150, sequenceid=318, filesize=30.5 K 2024-11-18T23:17:04,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/199dee39b8af4562946a435f050666b4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/199dee39b8af4562946a435f050666b4 2024-11-18T23:17:04,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/199dee39b8af4562946a435f050666b4, entries=150, sequenceid=318, filesize=12.0 K 2024-11-18T23:17:04,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/1a7a87fff34942198ba5eca19b79df26 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1a7a87fff34942198ba5eca19b79df26 2024-11-18T23:17:04,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1a7a87fff34942198ba5eca19b79df26, entries=150, sequenceid=318, filesize=12.0 K 2024-11-18T23:17:04,284 INFO [Thread-1441 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1441. 2024-11-18T23:17:04,284 DEBUG [Thread-1441 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:04,284 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:04,284 DEBUG [Thread-1441 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:04,285 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 656 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1967 rows 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 661 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1981 rows 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 657 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1964 rows 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 654 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1959 rows 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 657 2024-11-18T23:17:04,285 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1970 rows 2024-11-18T23:17:04,285 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:17:04,285 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:17:04,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:04,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:04,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:04,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=20.13 KB/20610 for 49690c5aec791d09014e5d76c85e7360 in 1013ms, sequenceid=318, compaction requested=true 2024-11-18T23:17:04,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:04,287 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 49690c5aec791d09014e5d76c85e7360:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:04,287 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:04,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:04,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T23:17:04,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:04,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T23:17:04,295 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:04,295 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/B is initiating minor compaction (all files) 2024-11-18T23:17:04,295 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/B in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,295 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4851102f6f644c2387fbba9b0d9180e7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/3235410af06e42b39219b925636fdf16, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/199dee39b8af4562946a435f050666b4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.8 K 2024-11-18T23:17:04,300 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4851102f6f644c2387fbba9b0d9180e7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731971818385 2024-11-18T23:17:04,301 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971824301"}]},"ts":"1731971824301"} 2024-11-18T23:17:04,304 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:04,304 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/A is initiating minor compaction (all files) 2024-11-18T23:17:04,304 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/A in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,305 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eb995c43b6a742089387d55661ca8d0b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/dbe87036bb2647ba9fa4cddffbe3d5f7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/91a906231eee439a88a39251a8aaa4c0] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=92.3 K 2024-11-18T23:17:04,305 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,305 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eb995c43b6a742089387d55661ca8d0b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/dbe87036bb2647ba9fa4cddffbe3d5f7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/91a906231eee439a88a39251a8aaa4c0] 2024-11-18T23:17:04,305 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3235410af06e42b39219b925636fdf16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731971819701 2024-11-18T23:17:04,306 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 199dee39b8af4562946a435f050666b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731971820992 2024-11-18T23:17:04,306 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb995c43b6a742089387d55661ca8d0b, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731971818385 2024-11-18T23:17:04,312 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T23:17:04,312 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T23:17:04,313 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbe87036bb2647ba9fa4cddffbe3d5f7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731971819701 2024-11-18T23:17:04,314 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91a906231eee439a88a39251a8aaa4c0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731971820992 2024-11-18T23:17:04,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:17:04,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, UNASSIGN}] 2024-11-18T23:17:04,335 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, UNASSIGN 2024-11-18T23:17:04,339 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:04,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, UNASSIGN because future has completed 2024-11-18T23:17:04,343 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:17:04,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:17:04,360 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:04,362 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#B#compaction#344 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:04,362 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e012e7e29c9845c692cd83d7605fc608 is 50, key is test_row_0/B:col10/1731971823268/Put/seqid=0 2024-11-18T23:17:04,365 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111856e52a37e3e346028db214d830a32cc8_49690c5aec791d09014e5d76c85e7360 store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:04,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742236_1412 (size=13153) 2024-11-18T23:17:04,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T23:17:04,410 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111856e52a37e3e346028db214d830a32cc8_49690c5aec791d09014e5d76c85e7360, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:04,410 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111856e52a37e3e346028db214d830a32cc8_49690c5aec791d09014e5d76c85e7360 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:04,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742237_1413 (size=4469) 2024-11-18T23:17:04,427 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#A#compaction#345 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:04,428 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/ebaaee9c72ed4793b3c88b1bc8bacdd3 is 175, key is test_row_0/A:col10/1731971823268/Put/seqid=0 2024-11-18T23:17:04,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742238_1414 (size=32107) 2024-11-18T23:17:04,465 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/ebaaee9c72ed4793b3c88b1bc8bacdd3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/ebaaee9c72ed4793b3c88b1bc8bacdd3 2024-11-18T23:17:04,484 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/A of 49690c5aec791d09014e5d76c85e7360 into ebaaee9c72ed4793b3c88b1bc8bacdd3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:04,484 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:04,484 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/A, priority=13, startTime=1731971824287; duration=0sec 2024-11-18T23:17:04,484 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:04,485 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:A 2024-11-18T23:17:04,485 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:04,485 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:04,486 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 49690c5aec791d09014e5d76c85e7360/C is initiating minor compaction (all files) 2024-11-18T23:17:04,486 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 49690c5aec791d09014e5d76c85e7360/C in TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,486 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/17fade7538cb4cd4843d65c084af9a4f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2b1f2225f07e4582b46c896795fdfa43, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1a7a87fff34942198ba5eca19b79df26] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp, totalSize=36.8 K 2024-11-18T23:17:04,486 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17fade7538cb4cd4843d65c084af9a4f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731971818385 2024-11-18T23:17:04,487 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b1f2225f07e4582b46c896795fdfa43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731971819701 2024-11-18T23:17:04,487 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a7a87fff34942198ba5eca19b79df26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731971820992 2024-11-18T23:17:04,499 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(122): Close 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:04,499 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:17:04,500 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1722): Closing 49690c5aec791d09014e5d76c85e7360, disabling compactions & flushes 2024-11-18T23:17:04,500 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,504 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 49690c5aec791d09014e5d76c85e7360#C#compaction#346 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:04,505 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/0a22ba88e6ed4e6aadfef5185277db18 is 50, key is test_row_0/C:col10/1731971823268/Put/seqid=0 2024-11-18T23:17:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742239_1415 (size=13153) 2024-11-18T23:17:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T23:17:04,819 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/e012e7e29c9845c692cd83d7605fc608 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e012e7e29c9845c692cd83d7605fc608 2024-11-18T23:17:04,838 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/B of 49690c5aec791d09014e5d76c85e7360 into e012e7e29c9845c692cd83d7605fc608(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:04,838 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:04,838 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/B, priority=13, startTime=1731971824287; duration=0sec 2024-11-18T23:17:04,838 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:04,838 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:B 2024-11-18T23:17:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T23:17:04,926 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/0a22ba88e6ed4e6aadfef5185277db18 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/0a22ba88e6ed4e6aadfef5185277db18 2024-11-18T23:17:04,936 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 49690c5aec791d09014e5d76c85e7360/C of 49690c5aec791d09014e5d76c85e7360 into 0a22ba88e6ed4e6aadfef5185277db18(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:04,936 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 49690c5aec791d09014e5d76c85e7360: 2024-11-18T23:17:04,936 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,936 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., storeName=49690c5aec791d09014e5d76c85e7360/C, priority=13, startTime=1731971824287; duration=0sec 2024-11-18T23:17:04,936 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,937 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. after waiting 1 ms 2024-11-18T23:17:04,937 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 49690c5aec791d09014e5d76c85e7360:C 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:04,937 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(2902): Flushing 49690c5aec791d09014e5d76c85e7360 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=A 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=B 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 49690c5aec791d09014e5d76c85e7360, store=C 2024-11-18T23:17:04,937 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:04,943 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188c6a224088944a7cba163dc6dcedeea7_49690c5aec791d09014e5d76c85e7360 is 50, key is test_row_2/A:col10/1731971824278/Put/seqid=0 2024-11-18T23:17:04,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742240_1416 (size=7374) 2024-11-18T23:17:05,358 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:05,364 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188c6a224088944a7cba163dc6dcedeea7_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188c6a224088944a7cba163dc6dcedeea7_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:05,365 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3f2b4861effb415a84b551986d2c40fe, store: [table=TestAcidGuarantees family=A region=49690c5aec791d09014e5d76c85e7360] 2024-11-18T23:17:05,366 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3f2b4861effb415a84b551986d2c40fe is 175, key is test_row_2/A:col10/1731971824278/Put/seqid=0 2024-11-18T23:17:05,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742241_1417 (size=13865) 2024-11-18T23:17:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T23:17:05,780 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3f2b4861effb415a84b551986d2c40fe 2024-11-18T23:17:05,798 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/514a1762f45346d68ff8ec4bca54c0da is 50, key is test_row_2/B:col10/1731971824278/Put/seqid=0 2024-11-18T23:17:05,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742242_1418 (size=7415) 2024-11-18T23:17:05,829 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/514a1762f45346d68ff8ec4bca54c0da 2024-11-18T23:17:05,862 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a6fe04b12ffb46f8a5be2fb5fa29ed37 is 50, key is test_row_2/C:col10/1731971824278/Put/seqid=0 2024-11-18T23:17:05,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742243_1419 (size=7415) 2024-11-18T23:17:06,301 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a6fe04b12ffb46f8a5be2fb5fa29ed37 2024-11-18T23:17:06,309 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/A/3f2b4861effb415a84b551986d2c40fe as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3f2b4861effb415a84b551986d2c40fe 2024-11-18T23:17:06,314 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3f2b4861effb415a84b551986d2c40fe, entries=50, sequenceid=327, filesize=13.5 K 2024-11-18T23:17:06,314 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/B/514a1762f45346d68ff8ec4bca54c0da as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/514a1762f45346d68ff8ec4bca54c0da 2024-11-18T23:17:06,321 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/514a1762f45346d68ff8ec4bca54c0da, entries=50, sequenceid=327, filesize=7.2 K 2024-11-18T23:17:06,322 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/.tmp/C/a6fe04b12ffb46f8a5be2fb5fa29ed37 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a6fe04b12ffb46f8a5be2fb5fa29ed37 2024-11-18T23:17:06,328 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a6fe04b12ffb46f8a5be2fb5fa29ed37, entries=50, sequenceid=327, filesize=7.2 K 2024-11-18T23:17:06,329 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 49690c5aec791d09014e5d76c85e7360 in 1392ms, sequenceid=327, compaction requested=false 2024-11-18T23:17:06,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/39b67c8699524a9388457af4c0f988be, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eebd02957b7741eda312206f91509f89, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a8cc128be99d4856881bfc7b793f6a32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/fb71ea9a069540c58f678b3f112ab5f5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/002616834d114e2d97388f61e814413b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cbc014664a3940a7a95ba08e1cf431ea, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/2165274606294e00a55d54631baf27cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3b405885b9904eaa8c0dcff36f6f7622, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f3146502761b435db3b8058276a5ed6e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f96c46d15e464347a74a5f6673efd0e1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/b380c8c5b7424255b6e5f66d716ff898, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e9e0781bae2f4495af5887b82053b0e1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/67df66a91b2a45a9a5be8b6b970cd9c5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4e0c3db0429d4b918459f0990a299f24, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cfc97859b31f4d69b63f39523d23d489, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a914f5e9521c4ad68af7d68598508b2b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e7518015aaf548b3a08b317e3846a950, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/04f3141538144aaaa5558860bb40f1b3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/6396e5e4aec042449cfe4b26dc12ebc5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/98c29ad0c5604a8dbf05b07ae34e490b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4d4318446985445c8ceb0891ef7846e0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/d2c74fac3f284d9c97cbe73c84987da3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eb995c43b6a742089387d55661ca8d0b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/dbe87036bb2647ba9fa4cddffbe3d5f7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/91a906231eee439a88a39251a8aaa4c0] to archive 2024-11-18T23:17:06,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:06,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/39b67c8699524a9388457af4c0f988be to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/39b67c8699524a9388457af4c0f988be 2024-11-18T23:17:06,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eebd02957b7741eda312206f91509f89 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eebd02957b7741eda312206f91509f89 2024-11-18T23:17:06,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a8cc128be99d4856881bfc7b793f6a32 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a8cc128be99d4856881bfc7b793f6a32 2024-11-18T23:17:06,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/fb71ea9a069540c58f678b3f112ab5f5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/fb71ea9a069540c58f678b3f112ab5f5 2024-11-18T23:17:06,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/002616834d114e2d97388f61e814413b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/002616834d114e2d97388f61e814413b 2024-11-18T23:17:06,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cbc014664a3940a7a95ba08e1cf431ea to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cbc014664a3940a7a95ba08e1cf431ea 2024-11-18T23:17:06,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/2165274606294e00a55d54631baf27cd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/2165274606294e00a55d54631baf27cd 2024-11-18T23:17:06,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3b405885b9904eaa8c0dcff36f6f7622 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3b405885b9904eaa8c0dcff36f6f7622 2024-11-18T23:17:06,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f3146502761b435db3b8058276a5ed6e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f3146502761b435db3b8058276a5ed6e 2024-11-18T23:17:06,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f96c46d15e464347a74a5f6673efd0e1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/f96c46d15e464347a74a5f6673efd0e1 2024-11-18T23:17:06,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/b380c8c5b7424255b6e5f66d716ff898 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/b380c8c5b7424255b6e5f66d716ff898 2024-11-18T23:17:06,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e9e0781bae2f4495af5887b82053b0e1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e9e0781bae2f4495af5887b82053b0e1 2024-11-18T23:17:06,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/67df66a91b2a45a9a5be8b6b970cd9c5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/67df66a91b2a45a9a5be8b6b970cd9c5 2024-11-18T23:17:06,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4e0c3db0429d4b918459f0990a299f24 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4e0c3db0429d4b918459f0990a299f24 2024-11-18T23:17:06,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cfc97859b31f4d69b63f39523d23d489 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/cfc97859b31f4d69b63f39523d23d489 2024-11-18T23:17:06,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a914f5e9521c4ad68af7d68598508b2b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/a914f5e9521c4ad68af7d68598508b2b 2024-11-18T23:17:06,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e7518015aaf548b3a08b317e3846a950 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/e7518015aaf548b3a08b317e3846a950 2024-11-18T23:17:06,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/04f3141538144aaaa5558860bb40f1b3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/04f3141538144aaaa5558860bb40f1b3 2024-11-18T23:17:06,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/6396e5e4aec042449cfe4b26dc12ebc5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/6396e5e4aec042449cfe4b26dc12ebc5 2024-11-18T23:17:06,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/98c29ad0c5604a8dbf05b07ae34e490b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/98c29ad0c5604a8dbf05b07ae34e490b 2024-11-18T23:17:06,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4d4318446985445c8ceb0891ef7846e0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/4d4318446985445c8ceb0891ef7846e0 2024-11-18T23:17:06,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T23:17:06,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/d2c74fac3f284d9c97cbe73c84987da3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/d2c74fac3f284d9c97cbe73c84987da3 2024-11-18T23:17:06,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eb995c43b6a742089387d55661ca8d0b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/eb995c43b6a742089387d55661ca8d0b 2024-11-18T23:17:06,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/dbe87036bb2647ba9fa4cddffbe3d5f7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/dbe87036bb2647ba9fa4cddffbe3d5f7 2024-11-18T23:17:06,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/91a906231eee439a88a39251a8aaa4c0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/91a906231eee439a88a39251a8aaa4c0 2024-11-18T23:17:06,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9ec5dfdc806649689a462431cd3c64be, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d426879c21274248b18e8c01684270e7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d67e21f7c8e749808d9ffee4a3841245, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1917c802ae39444c90b1bf6b292d2458, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1abc7ad6dacb4168ba3ddd43b2e94f90, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/0f58b67b53a44bddb60c10e1313ab05e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/b83725d5474341da8db90bd14478ccb0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/8b597cdc6a5e46749438a58eb7ecf75d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e758373b5e1148d68ae1428d1d3dc938, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ec422eac6e0848de99c84dfddc2b6c45, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e3821fc2c4d943499e15205d57037404, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a030a49d9434486eba7979cea857efc1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/65980a91612b4f0f8f8283f1f364d8af, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ae235dbaebc64ab3a96ee316ab147754, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4e1fd478929d4ef6b30224f3d305e48a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a6591bddaec34aa786d3cfaccc523ce4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/da395d69561a45c5a10be193ea11f664, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/57c4b390242e4fd89e0ef61764fd56fa, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/82f55d28c5ce4f43a983196360d624e2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/79a06a8365ad4eb88057dc808db3f51f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9e9a18b28d304d568fdc517b1ed397ba, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4851102f6f644c2387fbba9b0d9180e7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/91b114ad96634ad3a05b6dcad3a03033, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/3235410af06e42b39219b925636fdf16, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/199dee39b8af4562946a435f050666b4] to archive 2024-11-18T23:17:06,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:06,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9ec5dfdc806649689a462431cd3c64be to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9ec5dfdc806649689a462431cd3c64be 2024-11-18T23:17:06,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d426879c21274248b18e8c01684270e7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d426879c21274248b18e8c01684270e7 2024-11-18T23:17:06,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d67e21f7c8e749808d9ffee4a3841245 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/d67e21f7c8e749808d9ffee4a3841245 2024-11-18T23:17:06,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1917c802ae39444c90b1bf6b292d2458 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1917c802ae39444c90b1bf6b292d2458 2024-11-18T23:17:06,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1abc7ad6dacb4168ba3ddd43b2e94f90 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/1abc7ad6dacb4168ba3ddd43b2e94f90 2024-11-18T23:17:06,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/0f58b67b53a44bddb60c10e1313ab05e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/0f58b67b53a44bddb60c10e1313ab05e 2024-11-18T23:17:06,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/b83725d5474341da8db90bd14478ccb0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/b83725d5474341da8db90bd14478ccb0 2024-11-18T23:17:06,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/8b597cdc6a5e46749438a58eb7ecf75d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/8b597cdc6a5e46749438a58eb7ecf75d 2024-11-18T23:17:06,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e758373b5e1148d68ae1428d1d3dc938 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e758373b5e1148d68ae1428d1d3dc938 2024-11-18T23:17:06,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ec422eac6e0848de99c84dfddc2b6c45 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ec422eac6e0848de99c84dfddc2b6c45 2024-11-18T23:17:06,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e3821fc2c4d943499e15205d57037404 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e3821fc2c4d943499e15205d57037404 2024-11-18T23:17:06,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a030a49d9434486eba7979cea857efc1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a030a49d9434486eba7979cea857efc1 2024-11-18T23:17:06,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/65980a91612b4f0f8f8283f1f364d8af to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/65980a91612b4f0f8f8283f1f364d8af 2024-11-18T23:17:06,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ae235dbaebc64ab3a96ee316ab147754 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/ae235dbaebc64ab3a96ee316ab147754 2024-11-18T23:17:06,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4e1fd478929d4ef6b30224f3d305e48a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4e1fd478929d4ef6b30224f3d305e48a 2024-11-18T23:17:06,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a6591bddaec34aa786d3cfaccc523ce4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/a6591bddaec34aa786d3cfaccc523ce4 2024-11-18T23:17:06,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/da395d69561a45c5a10be193ea11f664 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/da395d69561a45c5a10be193ea11f664 2024-11-18T23:17:06,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/57c4b390242e4fd89e0ef61764fd56fa to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/57c4b390242e4fd89e0ef61764fd56fa 2024-11-18T23:17:06,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/82f55d28c5ce4f43a983196360d624e2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/82f55d28c5ce4f43a983196360d624e2 2024-11-18T23:17:06,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/79a06a8365ad4eb88057dc808db3f51f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/79a06a8365ad4eb88057dc808db3f51f 2024-11-18T23:17:06,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9e9a18b28d304d568fdc517b1ed397ba to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/9e9a18b28d304d568fdc517b1ed397ba 2024-11-18T23:17:06,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4851102f6f644c2387fbba9b0d9180e7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/4851102f6f644c2387fbba9b0d9180e7 2024-11-18T23:17:06,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/91b114ad96634ad3a05b6dcad3a03033 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/91b114ad96634ad3a05b6dcad3a03033 2024-11-18T23:17:06,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/3235410af06e42b39219b925636fdf16 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/3235410af06e42b39219b925636fdf16 2024-11-18T23:17:06,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/199dee39b8af4562946a435f050666b4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/199dee39b8af4562946a435f050666b4 2024-11-18T23:17:06,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/4151ce68498546f399d5a0030ca461b6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/902b1760c39946f7aee978bc53a6db8f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7815fd8040b64774ae10d610520a4772, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a57a7ef645ac4eaeaeaddc5110e20e61, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d64f8e77f3d240009a14e240cc5d4b83, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8b26eb7a49324ea5b0b84c3decb676ff, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d2acc9c767ab4e52ba4f33c8fe6343b6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/5b596daa5779478d8202543dedf11701, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/6c0904c3f0ae472c8ad9bb6cf085710a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/9f1dd569067842d6818e844b91cc28d4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7b2c2a4462464ad7b1fda9eae9b3958e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1989861927ea495da6db0866fb4a64bb, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/76e5d579dcc94f99a9a23cb18fdb4274, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/08247cd4706644b38b1d86817e782b3d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d8250a68c49b428a8b609426f41d2837, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/c5c14b3365b64bc5a84bb848dad3eca8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8f12dd45116149f7a07a41f1f0f52668, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a49a235431504e2598aa837a2085ec03, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2d50b9355f8e4a9b8a331a176af897f2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/60745606a7f7428a8374a7caf613ea67, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/17fade7538cb4cd4843d65c084af9a4f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/db1bf058369f4dc2b82ec71d0dac64f3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2b1f2225f07e4582b46c896795fdfa43, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1a7a87fff34942198ba5eca19b79df26] to archive 2024-11-18T23:17:06,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:06,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/3a90e8e3fbe9440ab4cb6ab3a894d2d3 2024-11-18T23:17:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/4151ce68498546f399d5a0030ca461b6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/4151ce68498546f399d5a0030ca461b6 2024-11-18T23:17:06,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/902b1760c39946f7aee978bc53a6db8f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/902b1760c39946f7aee978bc53a6db8f 2024-11-18T23:17:06,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7815fd8040b64774ae10d610520a4772 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7815fd8040b64774ae10d610520a4772 2024-11-18T23:17:06,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a57a7ef645ac4eaeaeaddc5110e20e61 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a57a7ef645ac4eaeaeaddc5110e20e61 2024-11-18T23:17:06,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d64f8e77f3d240009a14e240cc5d4b83 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d64f8e77f3d240009a14e240cc5d4b83 2024-11-18T23:17:06,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8b26eb7a49324ea5b0b84c3decb676ff to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8b26eb7a49324ea5b0b84c3decb676ff 2024-11-18T23:17:06,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d2acc9c767ab4e52ba4f33c8fe6343b6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d2acc9c767ab4e52ba4f33c8fe6343b6 2024-11-18T23:17:06,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/5b596daa5779478d8202543dedf11701 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/5b596daa5779478d8202543dedf11701 2024-11-18T23:17:06,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/6c0904c3f0ae472c8ad9bb6cf085710a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/6c0904c3f0ae472c8ad9bb6cf085710a 2024-11-18T23:17:06,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/9f1dd569067842d6818e844b91cc28d4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/9f1dd569067842d6818e844b91cc28d4 2024-11-18T23:17:06,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7b2c2a4462464ad7b1fda9eae9b3958e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/7b2c2a4462464ad7b1fda9eae9b3958e 2024-11-18T23:17:06,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1989861927ea495da6db0866fb4a64bb to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1989861927ea495da6db0866fb4a64bb 2024-11-18T23:17:06,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/76e5d579dcc94f99a9a23cb18fdb4274 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/76e5d579dcc94f99a9a23cb18fdb4274 2024-11-18T23:17:06,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/08247cd4706644b38b1d86817e782b3d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/08247cd4706644b38b1d86817e782b3d 2024-11-18T23:17:06,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d8250a68c49b428a8b609426f41d2837 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/d8250a68c49b428a8b609426f41d2837 2024-11-18T23:17:06,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/c5c14b3365b64bc5a84bb848dad3eca8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/c5c14b3365b64bc5a84bb848dad3eca8 2024-11-18T23:17:06,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8f12dd45116149f7a07a41f1f0f52668 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/8f12dd45116149f7a07a41f1f0f52668 2024-11-18T23:17:06,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a49a235431504e2598aa837a2085ec03 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a49a235431504e2598aa837a2085ec03 2024-11-18T23:17:06,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2d50b9355f8e4a9b8a331a176af897f2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2d50b9355f8e4a9b8a331a176af897f2 2024-11-18T23:17:06,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/60745606a7f7428a8374a7caf613ea67 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/60745606a7f7428a8374a7caf613ea67 2024-11-18T23:17:06,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/17fade7538cb4cd4843d65c084af9a4f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/17fade7538cb4cd4843d65c084af9a4f 2024-11-18T23:17:06,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/db1bf058369f4dc2b82ec71d0dac64f3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/db1bf058369f4dc2b82ec71d0dac64f3 2024-11-18T23:17:06,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2b1f2225f07e4582b46c896795fdfa43 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/2b1f2225f07e4582b46c896795fdfa43 2024-11-18T23:17:06,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1a7a87fff34942198ba5eca19b79df26 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/1a7a87fff34942198ba5eca19b79df26 2024-11-18T23:17:06,913 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/recovered.edits/330.seqid, newMaxSeqId=330, maxSeqId=4 2024-11-18T23:17:06,914 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. 2024-11-18T23:17:06,914 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1676): Region close journal for 49690c5aec791d09014e5d76c85e7360: Waiting for close lock at 1731971824499Running coprocessor pre-close hooks at 1731971824500 (+1 ms)Disabling compacts and flushes for region at 1731971824500Disabling writes for close at 1731971824937 (+437 ms)Obtaining lock to block concurrent updates at 1731971824937Preparing flush snapshotting stores in 49690c5aec791d09014e5d76c85e7360 at 1731971824937Finished memstore snapshotting TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731971824937Flushing stores of TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360. at 1731971824938 (+1 ms)Flushing 49690c5aec791d09014e5d76c85e7360/A: creating writer at 1731971824938Flushing mob file 49690c5aec791d09014e5d76c85e7360/A: appending metadata at 1731971824943 (+5 ms)Flushing mob file 49690c5aec791d09014e5d76c85e7360/A: closing flushed file at 1731971824943Flushing 49690c5aec791d09014e5d76c85e7360/A: appending metadata at 1731971825365 (+422 ms)Flushing 49690c5aec791d09014e5d76c85e7360/A: closing flushed file at 1731971825366 (+1 ms)Flushing 49690c5aec791d09014e5d76c85e7360/B: creating writer at 1731971825789 (+423 ms)Flushing 49690c5aec791d09014e5d76c85e7360/B: appending metadata at 1731971825796 (+7 ms)Flushing 49690c5aec791d09014e5d76c85e7360/B: closing flushed file at 1731971825796Flushing 49690c5aec791d09014e5d76c85e7360/C: creating writer at 1731971825845 (+49 ms)Flushing 49690c5aec791d09014e5d76c85e7360/C: appending metadata at 1731971825861 (+16 ms)Flushing 49690c5aec791d09014e5d76c85e7360/C: closing flushed file at 1731971825861Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@173f23ba: reopening flushed file at 1731971826308 (+447 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a9ada0c: reopening flushed file at 1731971826314 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b0fe91: reopening flushed file at 1731971826321 (+7 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 49690c5aec791d09014e5d76c85e7360 in 1392ms, sequenceid=327, compaction requested=false at 1731971826329 (+8 ms)Writing region close event to WAL at 1731971826910 (+581 ms)Running coprocessor post-close hooks at 1731971826914 (+4 ms)Closed at 1731971826914 2024-11-18T23:17:06,916 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(157): Closed 49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:06,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=49690c5aec791d09014e5d76c85e7360, regionState=CLOSED 2024-11-18T23:17:06,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:17:06,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-18T23:17:06,924 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseRegionProcedure 49690c5aec791d09014e5d76c85e7360, server=9d96596e55a9,32899,1731971713394 in 2.5790 sec 2024-11-18T23:17:06,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-18T23:17:06,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=49690c5aec791d09014e5d76c85e7360, UNASSIGN in 2.6010 sec 2024-11-18T23:17:06,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-18T23:17:06,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6120 sec 2024-11-18T23:17:06,939 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971826939"}]},"ts":"1731971826939"} 2024-11-18T23:17:06,946 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T23:17:06,947 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T23:17:06,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6610 sec 2024-11-18T23:17:08,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-18T23:17:08,458 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:08,459 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T23:17:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:08,462 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T23:17:08,463 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=103, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:08,472 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,510 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/recovered.edits] 2024-11-18T23:17:08,544 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3f2b4861effb415a84b551986d2c40fe to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/3f2b4861effb415a84b551986d2c40fe 2024-11-18T23:17:08,552 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/ebaaee9c72ed4793b3c88b1bc8bacdd3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/A/ebaaee9c72ed4793b3c88b1bc8bacdd3 2024-11-18T23:17:08,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T23:17:08,581 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/514a1762f45346d68ff8ec4bca54c0da to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/514a1762f45346d68ff8ec4bca54c0da 2024-11-18T23:17:08,586 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e012e7e29c9845c692cd83d7605fc608 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/B/e012e7e29c9845c692cd83d7605fc608 2024-11-18T23:17:08,591 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/0a22ba88e6ed4e6aadfef5185277db18 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/0a22ba88e6ed4e6aadfef5185277db18 2024-11-18T23:17:08,593 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a6fe04b12ffb46f8a5be2fb5fa29ed37 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/C/a6fe04b12ffb46f8a5be2fb5fa29ed37 2024-11-18T23:17:08,602 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/recovered.edits/330.seqid to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360/recovered.edits/330.seqid 2024-11-18T23:17:08,603 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,603 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T23:17:08,608 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T23:17:08,610 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T23:17:08,617 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807e9b0ea1a94415284c98b93c818876f_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111807e9b0ea1a94415284c98b93c818876f_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,618 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118096799882a994d6588aef753b5c4cd55_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118096799882a994d6588aef753b5c4cd55_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,619 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a66c23ac2724bec8301525bda6020a8_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181a66c23ac2724bec8301525bda6020a8_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,624 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118285772b3b64b4875b813b3fef8162561_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118285772b3b64b4875b813b3fef8162561_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,630 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182b0fbf2f76ce463abcb1659dcd119828_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411182b0fbf2f76ce463abcb1659dcd119828_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,642 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118570c548b69b5490d975a28c90d1e8f9f_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118570c548b69b5490d975a28c90d1e8f9f_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,656 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118807220dbfb7e489e8a7311f2ed42749f_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118807220dbfb7e489e8a7311f2ed42749f_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,673 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111880b2850022664bebb5fbd5cd4d2a4778_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111880b2850022664bebb5fbd5cd4d2a4778_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,682 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188c6a224088944a7cba163dc6dcedeea7_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188c6a224088944a7cba163dc6dcedeea7_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,693 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891d3a7d22026434884b91ce924f7c012_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111891d3a7d22026434884b91ce924f7c012_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,699 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189be369aa3ccf40ffb5f9575be71398c6_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189be369aa3ccf40ffb5f9575be71398c6_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,705 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189f32bf86b2b04cd5a2ac6f07fbf5a0d4_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411189f32bf86b2b04cd5a2ac6f07fbf5a0d4_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,720 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b616b0ff25ba42158bfe400fe485abd1_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b616b0ff25ba42158bfe400fe485abd1_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,741 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118da9536d04bac4da9b520536b51afe8b9_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118da9536d04bac4da9b520536b51afe8b9_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,757 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ebfc605f4181415b95a5adb5039f3a82_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ebfc605f4181415b95a5adb5039f3a82_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,778 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f76b100eabe1465a8eedcc044272ff8c_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f76b100eabe1465a8eedcc044272ff8c_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T23:17:08,785 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f852be67b8c8486a8ef514f2e1d907af_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f852be67b8c8486a8ef514f2e1d907af_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,798 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9a1ee0aeb7b4835b6173ef680be5230_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118f9a1ee0aeb7b4835b6173ef680be5230_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,802 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fcc79e999b9d4a2a8330a39858a6aedd_49690c5aec791d09014e5d76c85e7360 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118fcc79e999b9d4a2a8330a39858a6aedd_49690c5aec791d09014e5d76c85e7360 2024-11-18T23:17:08,803 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T23:17:08,812 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=103, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:08,824 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T23:17:08,836 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T23:17:08,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=103, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:08,839 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T23:17:08,839 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731971828839"}]},"ts":"9223372036854775807"} 2024-11-18T23:17:08,844 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T23:17:08,844 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 49690c5aec791d09014e5d76c85e7360, NAME => 'TestAcidGuarantees,,1731971798497.49690c5aec791d09014e5d76c85e7360.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T23:17:08,844 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T23:17:08,845 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731971828845"}]},"ts":"9223372036854775807"} 2024-11-18T23:17:08,857 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T23:17:08,858 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=103, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:08,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 399 msec 2024-11-18T23:17:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-18T23:17:09,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T23:17:09,088 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:09,101 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1665 (was 1443) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6543 (was 7509) 2024-11-18T23:17:09,113 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=1665, ProcessCount=11, AvailableMemoryMB=6543 2024-11-18T23:17:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:17:09,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:17:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:09,119 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:17:09,119 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:09,121 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 104 2024-11-18T23:17:09,122 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:17:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-18T23:17:09,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742244_1420 (size=1061) 2024-11-18T23:17:09,182 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:17:09,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742245_1421 (size=53) 2024-11-18T23:17:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-18T23:17:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-18T23:17:09,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:17:09,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 37a0e116df39ce62ed45cb517bbc0fb6, disabling compactions & flushes 2024-11-18T23:17:09,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:09,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:09,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. after waiting 0 ms 2024-11-18T23:17:09,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:09,628 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:09,628 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 37a0e116df39ce62ed45cb517bbc0fb6: Waiting for close lock at 1731971829628Disabling compacts and flushes for region at 1731971829628Disabling writes for close at 1731971829628Writing region close event to WAL at 1731971829628Closed at 1731971829628 2024-11-18T23:17:09,629 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:17:09,630 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731971829629"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731971829629"}]},"ts":"1731971829629"} 2024-11-18T23:17:09,632 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:17:09,633 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:17:09,633 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971829633"}]},"ts":"1731971829633"} 2024-11-18T23:17:09,635 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T23:17:09,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, ASSIGN}] 2024-11-18T23:17:09,637 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, ASSIGN 2024-11-18T23:17:09,638 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, ASSIGN; state=OFFLINE, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=false 2024-11-18T23:17:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-18T23:17:09,790 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=37a0e116df39ce62ed45cb517bbc0fb6, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:09,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, ASSIGN because future has completed 2024-11-18T23:17:09,798 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:17:09,985 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:09,986 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:17:09,986 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:09,987 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:17:09,987 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:09,987 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,004 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,020 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:10,021 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37a0e116df39ce62ed45cb517bbc0fb6 columnFamilyName A 2024-11-18T23:17:10,021 DEBUG [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:10,024 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.HStore(327): Store=37a0e116df39ce62ed45cb517bbc0fb6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:10,024 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,039 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:10,039 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37a0e116df39ce62ed45cb517bbc0fb6 columnFamilyName B 2024-11-18T23:17:10,039 DEBUG [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:10,044 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.HStore(327): Store=37a0e116df39ce62ed45cb517bbc0fb6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:10,044 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,052 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:10,053 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37a0e116df39ce62ed45cb517bbc0fb6 columnFamilyName C 2024-11-18T23:17:10,053 DEBUG [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:10,068 INFO [StoreOpener-37a0e116df39ce62ed45cb517bbc0fb6-1 {}] regionserver.HStore(327): Store=37a0e116df39ce62ed45cb517bbc0fb6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:10,068 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,068 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,080 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,085 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,096 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,096 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,100 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:17:10,111 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,122 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:17:10,123 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 37a0e116df39ce62ed45cb517bbc0fb6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64277934, jitterRate=-0.042184144258499146}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:17:10,123 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,124 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 37a0e116df39ce62ed45cb517bbc0fb6: Running coprocessor pre-open hook at 1731971829987Writing region info on filesystem at 1731971829987Initializing all the Stores at 1731971829996 (+9 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971829997 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971830003 (+6 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971830003Cleaning up temporary data from old regions at 1731971830096 (+93 ms)Running coprocessor post-open hooks at 1731971830123 (+27 ms)Region opened successfully at 1731971830124 (+1 ms) 2024-11-18T23:17:10,134 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., pid=106, masterSystemTime=1731971829960 2024-11-18T23:17:10,140 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,140 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,144 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=37a0e116df39ce62ed45cb517bbc0fb6, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:17:10,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-18T23:17:10,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; OpenRegionProcedure 37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 in 363 msec 2024-11-18T23:17:10,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-18T23:17:10,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, ASSIGN in 534 msec 2024-11-18T23:17:10,176 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:17:10,176 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971830176"}]},"ts":"1731971830176"} 2024-11-18T23:17:10,184 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T23:17:10,188 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:17:10,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.0720 sec 2024-11-18T23:17:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-18T23:17:10,268 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:10,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,279 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e4fdf82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,280 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,281 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,282 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,283 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f24e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,283 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,285 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,290 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:17:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,42339,1731971712494 2024-11-18T23:17:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3761751f 2024-11-18T23:17:10,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:17:10,294 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:17:10,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,299 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c1ae4c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,308 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d12e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,324 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149d34b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,327 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405665b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,333 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,333 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,336 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a59f8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e7bff6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,354 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,355 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,360 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,364 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b7d7c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,372 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,372 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,376 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22e57560, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,378 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ca71908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,392 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18eeb120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,394 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,397 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,398 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25715ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,402 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,407 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,407 DEBUG [Time-limited test {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,409 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22a2f7cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,415 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,419 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,420 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@283a75dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbd92cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c1b90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,436 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,444 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b98380e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,451 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e6829a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658b2e69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,453 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:10,453 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:10,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ac52ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:10,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:10,459 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:10,464 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed7bfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:10,467 DEBUG [Thread-1909 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,468 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,469 DEBUG [Thread-1918 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,470 DEBUG [Thread-1903 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,472 DEBUG [Thread-1903 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,472 DEBUG [Thread-1903 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36716, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,478 DEBUG [Thread-1911 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,481 DEBUG [Thread-1907 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,483 DEBUG [Thread-1920 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,484 DEBUG [Thread-1922 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,485 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:10,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,485 DEBUG [Thread-1922 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,485 DEBUG [Thread-1920 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-18T23:17:10,486 DEBUG [Thread-1920 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,487 DEBUG [Thread-1905 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T23:17:10,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,487 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,488 DEBUG [Thread-1922 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,492 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,497 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:10,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:10,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,502 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:10,504 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:10,512 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:10,513 DEBUG [Thread-1922 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:10,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2] 2024-11-18T23:17:10,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:17:10,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:10,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:10,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:10,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:10,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:10,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:10,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/319462f09ebe4a8c80b0fbf09357c4ec is 50, key is test_row_0/A:col10/1731971830503/Put/seqid=0 2024-11-18T23:17:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T23:17:10,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742246_1422 (size=12001) 2024-11-18T23:17:10,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/319462f09ebe4a8c80b0fbf09357c4ec 2024-11-18T23:17:10,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971830595, max timestamp=1731971830657Num uniques 859; 2024-11-18T23:17:10,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:10,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T23:17:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971830595, max timestamp=1731971830657Num uniques 859; 2024-11-18T23:17:10,669 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:10,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971830595, max timestamp=1731971830657Num uniques 859; 2024-11-18T23:17:10,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:10,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/13ff9abf4244457f8e948f6dbab5d097 is 50, key is test_row_0/B:col10/1731971830503/Put/seqid=0 2024-11-18T23:17:10,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742247_1423 (size=12001) 2024-11-18T23:17:10,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/13ff9abf4244457f8e948f6dbab5d097 2024-11-18T23:17:10,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/2cdca9f2e0cf4ed1b792732ddaa044bc is 50, key is test_row_0/C:col10/1731971830503/Put/seqid=0 2024-11-18T23:17:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971890767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971890771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,806 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T23:17:10,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T23:17:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:10,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971890794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971890802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971890805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742248_1424 (size=12001) 2024-11-18T23:17:10,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971890918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971890918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971890948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971890948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:10,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971890952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:10,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:10,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T23:17:10,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:10,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:10,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:10,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T23:17:11,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971891148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T23:17:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971891155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:11,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:11,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:11,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:11,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971891168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971891168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971891188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,256 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/2cdca9f2e0cf4ed1b792732ddaa044bc 2024-11-18T23:17:11,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/319462f09ebe4a8c80b0fbf09357c4ec as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/319462f09ebe4a8c80b0fbf09357c4ec 2024-11-18T23:17:11,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T23:17:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:11,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:11,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:11,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:11,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:11,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/319462f09ebe4a8c80b0fbf09357c4ec, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T23:17:11,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/13ff9abf4244457f8e948f6dbab5d097 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/13ff9abf4244457f8e948f6dbab5d097 2024-11-18T23:17:11,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/13ff9abf4244457f8e948f6dbab5d097, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T23:17:11,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/2cdca9f2e0cf4ed1b792732ddaa044bc as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2cdca9f2e0cf4ed1b792732ddaa044bc 2024-11-18T23:17:11,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2cdca9f2e0cf4ed1b792732ddaa044bc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-18T23:17:11,396 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 37a0e116df39ce62ed45cb517bbc0fb6 in 878ms, sequenceid=12, compaction requested=false 2024-11-18T23:17:11,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:11,468 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T23:17:11,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-18T23:17:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:11,485 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T23:17:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:11,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:11,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:11,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:11,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/e457f8356d7b4383907934d46b2a6b9b is 50, key is test_row_0/A:col10/1731971830793/Put/seqid=0 2024-11-18T23:17:11,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971891542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971891545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,562 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971891553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971891560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971891553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742249_1425 (size=12001) 2024-11-18T23:17:11,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T23:17:11,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971891668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971891669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971891688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971891688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971891688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,707 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971891900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,921 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971891900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971891921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971891922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971891922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:11,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:11,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/e457f8356d7b4383907934d46b2a6b9b 2024-11-18T23:17:12,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/d1e835afabce4b21b6c74535bfbc2220 is 50, key is test_row_0/B:col10/1731971830793/Put/seqid=0 2024-11-18T23:17:12,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742250_1426 (size=12001) 2024-11-18T23:17:12,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971892228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971892236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971892257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971892258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971892260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,477 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:17:12,517 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/d1e835afabce4b21b6c74535bfbc2220 2024-11-18T23:17:12,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/76e0bdbbccb54f9d849106ce62d2007d is 50, key is test_row_0/C:col10/1731971830793/Put/seqid=0 2024-11-18T23:17:12,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742251_1427 (size=12001) 2024-11-18T23:17:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T23:17:12,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971892777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971892777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971892798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971892798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971892798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:12,997 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/76e0bdbbccb54f9d849106ce62d2007d 2024-11-18T23:17:13,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/e457f8356d7b4383907934d46b2a6b9b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e457f8356d7b4383907934d46b2a6b9b 2024-11-18T23:17:13,048 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e457f8356d7b4383907934d46b2a6b9b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T23:17:13,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/d1e835afabce4b21b6c74535bfbc2220 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/d1e835afabce4b21b6c74535bfbc2220 2024-11-18T23:17:13,055 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/d1e835afabce4b21b6c74535bfbc2220, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T23:17:13,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/76e0bdbbccb54f9d849106ce62d2007d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/76e0bdbbccb54f9d849106ce62d2007d 2024-11-18T23:17:13,064 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/76e0bdbbccb54f9d849106ce62d2007d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-18T23:17:13,068 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1582ms, sequenceid=49, compaction requested=false 2024-11-18T23:17:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:13,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-18T23:17:13,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-18T23:17:13,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-18T23:17:13,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5730 sec 2024-11-18T23:17:13,086 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.6000 sec 2024-11-18T23:17:13,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T23:17:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:13,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:13,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/ccbcb9fa7a994c5da4f32714e815ee02 is 50, key is test_row_0/A:col10/1731971833799/Put/seqid=0 2024-11-18T23:17:13,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742252_1428 (size=14341) 2024-11-18T23:17:13,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:13,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971893899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:13,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971893901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:13,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971893901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:13,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971893903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:13,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971893903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:13,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971894018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,042 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971894028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971894028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971894028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971894040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/ccbcb9fa7a994c5da4f32714e815ee02 2024-11-18T23:17:14,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971894252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,262 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971894267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/fac6b62083d34370ac05966de3d850a8 is 50, key is test_row_0/B:col10/1731971833799/Put/seqid=0 2024-11-18T23:17:14,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971894267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971894267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971894267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,301 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742253_1429 (size=12001) 2024-11-18T23:17:14,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/fac6b62083d34370ac05966de3d850a8 2024-11-18T23:17:14,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/96c1ae58ff7543989353a69ae2ddd321 is 50, key is test_row_0/C:col10/1731971833799/Put/seqid=0 2024-11-18T23:17:14,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742254_1430 (size=12001) 2024-11-18T23:17:14,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/96c1ae58ff7543989353a69ae2ddd321 2024-11-18T23:17:14,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/ccbcb9fa7a994c5da4f32714e815ee02 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ccbcb9fa7a994c5da4f32714e815ee02 2024-11-18T23:17:14,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ccbcb9fa7a994c5da4f32714e815ee02, entries=200, sequenceid=61, filesize=14.0 K 2024-11-18T23:17:14,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/fac6b62083d34370ac05966de3d850a8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/fac6b62083d34370ac05966de3d850a8 2024-11-18T23:17:14,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971894572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/fac6b62083d34370ac05966de3d850a8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:17:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,612 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971894608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971894608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971894610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/96c1ae58ff7543989353a69ae2ddd321 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/96c1ae58ff7543989353a69ae2ddd321 2024-11-18T23:17:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:14,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971894612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:14,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:14,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-18T23:17:14,661 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:14,661 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:14,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-18T23:17:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T23:17:14,669 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:14,670 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:14,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:14,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/96c1ae58ff7543989353a69ae2ddd321, entries=150, sequenceid=61, filesize=11.7 K 2024-11-18T23:17:14,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 37a0e116df39ce62ed45cb517bbc0fb6 in 901ms, sequenceid=61, compaction requested=true 2024-11-18T23:17:14,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:14,705 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:14,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:14,706 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:14,706 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/A is initiating minor compaction (all files) 2024-11-18T23:17:14,706 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/A in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:14,706 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/319462f09ebe4a8c80b0fbf09357c4ec, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e457f8356d7b4383907934d46b2a6b9b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ccbcb9fa7a994c5da4f32714e815ee02] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=37.4 K 2024-11-18T23:17:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:14,706 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 319462f09ebe4a8c80b0fbf09357c4ec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731971830503 2024-11-18T23:17:14,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:14,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:14,707 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:14,709 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e457f8356d7b4383907934d46b2a6b9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731971830789 2024-11-18T23:17:14,716 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccbcb9fa7a994c5da4f32714e815ee02, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971831541 2024-11-18T23:17:14,721 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:14,721 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/B is initiating minor compaction (all files) 2024-11-18T23:17:14,721 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/B in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:14,721 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/13ff9abf4244457f8e948f6dbab5d097, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/d1e835afabce4b21b6c74535bfbc2220, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/fac6b62083d34370ac05966de3d850a8] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=35.2 K 2024-11-18T23:17:14,725 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 13ff9abf4244457f8e948f6dbab5d097, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731971830503 2024-11-18T23:17:14,728 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e835afabce4b21b6c74535bfbc2220, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731971830789 2024-11-18T23:17:14,735 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting fac6b62083d34370ac05966de3d850a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971831541 2024-11-18T23:17:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T23:17:14,798 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#A#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:14,799 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/ca0801d504cc4639be7a957eb3765516 is 50, key is test_row_0/A:col10/1731971833799/Put/seqid=0 2024-11-18T23:17:14,830 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:14,831 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/ee09733345064e97a3299e9e1000ee45 is 50, key is test_row_0/B:col10/1731971833799/Put/seqid=0 2024-11-18T23:17:14,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-18T23:17:14,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:14,833 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-18T23:17:14,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:14,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:14,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:14,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742255_1431 (size=12104) 2024-11-18T23:17:14,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fbaac120514e44cc80346b023b29c4f1 is 50, key is test_row_0/A:col10/1731971833901/Put/seqid=0 2024-11-18T23:17:14,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742256_1432 (size=12104) 2024-11-18T23:17:14,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742257_1433 (size=12001) 2024-11-18T23:17:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T23:17:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:15,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:15,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971895189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971895193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971895195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971895196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971895197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,303 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/ca0801d504cc4639be7a957eb3765516 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ca0801d504cc4639be7a957eb3765516 2024-11-18T23:17:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T23:17:15,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971895317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,337 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/ee09733345064e97a3299e9e1000ee45 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ee09733345064e97a3299e9e1000ee45 2024-11-18T23:17:15,338 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/A of 37a0e116df39ce62ed45cb517bbc0fb6 into ca0801d504cc4639be7a957eb3765516(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:15,338 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:15,338 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/A, priority=13, startTime=1731971834705; duration=0sec 2024-11-18T23:17:15,338 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:15,338 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:A 2024-11-18T23:17:15,338 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971895328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,347 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971895328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,347 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/C is initiating minor compaction (all files) 2024-11-18T23:17:15,347 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/C in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:15,347 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2cdca9f2e0cf4ed1b792732ddaa044bc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/76e0bdbbccb54f9d849106ce62d2007d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/96c1ae58ff7543989353a69ae2ddd321] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=35.2 K 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971895328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971895328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,350 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fbaac120514e44cc80346b023b29c4f1 2024-11-18T23:17:15,350 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cdca9f2e0cf4ed1b792732ddaa044bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731971830503 2024-11-18T23:17:15,352 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76e0bdbbccb54f9d849106ce62d2007d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731971830789 2024-11-18T23:17:15,357 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96c1ae58ff7543989353a69ae2ddd321, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971831541 2024-11-18T23:17:15,361 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/B of 37a0e116df39ce62ed45cb517bbc0fb6 into ee09733345064e97a3299e9e1000ee45(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:15,361 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:15,361 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/B, priority=13, startTime=1731971834706; duration=0sec 2024-11-18T23:17:15,361 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:15,361 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:B 2024-11-18T23:17:15,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/c15ce5d2f901471090f6023fdeae4bcf is 50, key is test_row_0/B:col10/1731971833901/Put/seqid=0 2024-11-18T23:17:15,433 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:15,434 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/bb1feb17120f472d8d96ed36d435c50c is 50, key is test_row_0/C:col10/1731971833799/Put/seqid=0 2024-11-18T23:17:15,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742258_1434 (size=12001) 2024-11-18T23:17:15,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742259_1435 (size=12104) 2024-11-18T23:17:15,535 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/bb1feb17120f472d8d96ed36d435c50c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bb1feb17120f472d8d96ed36d435c50c 2024-11-18T23:17:15,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971895527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,537 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,563 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/C of 37a0e116df39ce62ed45cb517bbc0fb6 into bb1feb17120f472d8d96ed36d435c50c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:15,563 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:15,563 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/C, priority=13, startTime=1731971834707; duration=0sec 2024-11-18T23:17:15,563 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:15,563 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:C 2024-11-18T23:17:15,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971895558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971895558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971895558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971895558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T23:17:15,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971895848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,857 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/c15ce5d2f901471090f6023fdeae4bcf 2024-11-18T23:17:15,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/e8cae4cc22e445ca93c11d7aa23d9824 is 50, key is test_row_0/C:col10/1731971833901/Put/seqid=0 2024-11-18T23:17:15,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971895881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971895881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971895881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971895888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:15,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742260_1436 (size=12001) 2024-11-18T23:17:15,912 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/e8cae4cc22e445ca93c11d7aa23d9824 2024-11-18T23:17:15,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fbaac120514e44cc80346b023b29c4f1 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fbaac120514e44cc80346b023b29c4f1 2024-11-18T23:17:15,938 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fbaac120514e44cc80346b023b29c4f1, entries=150, sequenceid=81, filesize=11.7 K 2024-11-18T23:17:15,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/c15ce5d2f901471090f6023fdeae4bcf as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/c15ce5d2f901471090f6023fdeae4bcf 2024-11-18T23:17:15,951 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/c15ce5d2f901471090f6023fdeae4bcf, entries=150, sequenceid=81, filesize=11.7 K 2024-11-18T23:17:15,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/e8cae4cc22e445ca93c11d7aa23d9824 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e8cae4cc22e445ca93c11d7aa23d9824 2024-11-18T23:17:15,985 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e8cae4cc22e445ca93c11d7aa23d9824, entries=150, sequenceid=81, filesize=11.7 K 2024-11-18T23:17:16,012 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1179ms, sequenceid=81, compaction requested=false 2024-11-18T23:17:16,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:16,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:16,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-18T23:17:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-18T23:17:16,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.3710 sec 2024-11-18T23:17:16,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-18T23:17:16,036 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3580 sec 2024-11-18T23:17:16,291 WARN [regionserver/9d96596e55a9:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-18T23:17:16,292 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/319462f09ebe4a8c80b0fbf09357c4ec, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e457f8356d7b4383907934d46b2a6b9b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ccbcb9fa7a994c5da4f32714e815ee02] to archive 2024-11-18T23:17:16,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:16,302 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/319462f09ebe4a8c80b0fbf09357c4ec to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/319462f09ebe4a8c80b0fbf09357c4ec 2024-11-18T23:17:16,304 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e457f8356d7b4383907934d46b2a6b9b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e457f8356d7b4383907934d46b2a6b9b 2024-11-18T23:17:16,305 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ccbcb9fa7a994c5da4f32714e815ee02 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ccbcb9fa7a994c5da4f32714e815ee02 2024-11-18T23:17:16,306 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/13ff9abf4244457f8e948f6dbab5d097, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/d1e835afabce4b21b6c74535bfbc2220, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/fac6b62083d34370ac05966de3d850a8] to archive 2024-11-18T23:17:16,307 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:16,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/13ff9abf4244457f8e948f6dbab5d097 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/13ff9abf4244457f8e948f6dbab5d097 2024-11-18T23:17:16,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/d1e835afabce4b21b6c74535bfbc2220 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/d1e835afabce4b21b6c74535bfbc2220 2024-11-18T23:17:16,312 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/fac6b62083d34370ac05966de3d850a8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/fac6b62083d34370ac05966de3d850a8 2024-11-18T23:17:16,313 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2cdca9f2e0cf4ed1b792732ddaa044bc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/76e0bdbbccb54f9d849106ce62d2007d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/96c1ae58ff7543989353a69ae2ddd321] to archive 2024-11-18T23:17:16,314 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:16,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2cdca9f2e0cf4ed1b792732ddaa044bc to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2cdca9f2e0cf4ed1b792732ddaa044bc 2024-11-18T23:17:16,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/76e0bdbbccb54f9d849106ce62d2007d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/76e0bdbbccb54f9d849106ce62d2007d 2024-11-18T23:17:16,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9d96596e55a9:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/96c1ae58ff7543989353a69ae2ddd321 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/96c1ae58ff7543989353a69ae2ddd321 2024-11-18T23:17:16,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:16,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:17:16,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:16,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:16,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:16,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/70000a3059464dd8bb11bc8f10730b9e is 50, key is test_row_0/A:col10/1731971835177/Put/seqid=0 2024-11-18T23:17:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742261_1437 (size=12001) 2024-11-18T23:17:16,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/70000a3059464dd8bb11bc8f10730b9e 2024-11-18T23:17:16,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e829a956af6248fdb85f44145cfd2794 is 50, key is test_row_0/B:col10/1731971835177/Put/seqid=0 2024-11-18T23:17:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971896446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971896450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971896453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971896458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971896461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742262_1438 (size=12001) 2024-11-18T23:17:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971896580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971896580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971896580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971896580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971896580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971896799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971896799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971896805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971896805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:16,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971896809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:16,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:16,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-18T23:17:16,827 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:16,828 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-18T23:17:16,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T23:17:16,831 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:16,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:16,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e829a956af6248fdb85f44145cfd2794 2024-11-18T23:17:16,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/24d4f5f1b5144f8d8b553e36c7bf92d8 is 50, key is test_row_0/C:col10/1731971835177/Put/seqid=0 2024-11-18T23:17:16,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T23:17:16,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742263_1439 (size=12001) 2024-11-18T23:17:16,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-18T23:17:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:16,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:16,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:17,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971897129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971897129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-18T23:17:17,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:17,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:17,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971897130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971897132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971897138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T23:17:17,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-18T23:17:17,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:17,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:17,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/24d4f5f1b5144f8d8b553e36c7bf92d8 2024-11-18T23:17:17,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/70000a3059464dd8bb11bc8f10730b9e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/70000a3059464dd8bb11bc8f10730b9e 2024-11-18T23:17:17,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/70000a3059464dd8bb11bc8f10730b9e, entries=150, sequenceid=97, filesize=11.7 K 2024-11-18T23:17:17,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e829a956af6248fdb85f44145cfd2794 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e829a956af6248fdb85f44145cfd2794 2024-11-18T23:17:17,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e829a956af6248fdb85f44145cfd2794, entries=150, sequenceid=97, filesize=11.7 K 2024-11-18T23:17:17,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/24d4f5f1b5144f8d8b553e36c7bf92d8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/24d4f5f1b5144f8d8b553e36c7bf92d8 2024-11-18T23:17:17,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/24d4f5f1b5144f8d8b553e36c7bf92d8, entries=150, sequenceid=97, filesize=11.7 K 2024-11-18T23:17:17,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 37a0e116df39ce62ed45cb517bbc0fb6 in 998ms, sequenceid=97, compaction requested=true 2024-11-18T23:17:17,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:17,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:17,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:17,382 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:17,382 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:17,383 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:17,383 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/A is initiating minor compaction (all files) 2024-11-18T23:17:17,383 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/A in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,383 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ca0801d504cc4639be7a957eb3765516, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fbaac120514e44cc80346b023b29c4f1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/70000a3059464dd8bb11bc8f10730b9e] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=35.3 K 2024-11-18T23:17:17,383 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0801d504cc4639be7a957eb3765516, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971831541 2024-11-18T23:17:17,384 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:17,384 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/B is initiating minor compaction (all files) 2024-11-18T23:17:17,384 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbaac120514e44cc80346b023b29c4f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731971833894 2024-11-18T23:17:17,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/B in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ee09733345064e97a3299e9e1000ee45, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/c15ce5d2f901471090f6023fdeae4bcf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e829a956af6248fdb85f44145cfd2794] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=35.3 K 2024-11-18T23:17:17,384 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70000a3059464dd8bb11bc8f10730b9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731971835177 2024-11-18T23:17:17,384 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ee09733345064e97a3299e9e1000ee45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971831541 2024-11-18T23:17:17,385 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c15ce5d2f901471090f6023fdeae4bcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731971833894 2024-11-18T23:17:17,385 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e829a956af6248fdb85f44145cfd2794, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731971835177 2024-11-18T23:17:17,399 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#A#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:17,400 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/6c71ce65064b41f7ae841e78669c6059 is 50, key is test_row_0/A:col10/1731971835177/Put/seqid=0 2024-11-18T23:17:17,426 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#B#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:17,427 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/92dd4d6e288d4bc7af0d11c288310701 is 50, key is test_row_0/B:col10/1731971835177/Put/seqid=0 2024-11-18T23:17:17,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742264_1440 (size=12104) 2024-11-18T23:17:17,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742265_1441 (size=12104) 2024-11-18T23:17:17,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-18T23:17:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T23:17:17,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:17,454 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/6c71ce65064b41f7ae841e78669c6059 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/6c71ce65064b41f7ae841e78669c6059 2024-11-18T23:17:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T23:17:17,458 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/92dd4d6e288d4bc7af0d11c288310701 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/92dd4d6e288d4bc7af0d11c288310701 2024-11-18T23:17:17,462 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/A of 37a0e116df39ce62ed45cb517bbc0fb6 into 6c71ce65064b41f7ae841e78669c6059(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:17,462 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:17,462 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/A, priority=13, startTime=1731971837382; duration=0sec 2024-11-18T23:17:17,462 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:17,462 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:A 2024-11-18T23:17:17,462 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:17,466 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/B of 37a0e116df39ce62ed45cb517bbc0fb6 into 92dd4d6e288d4bc7af0d11c288310701(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:17,466 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:17,466 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/B, priority=13, startTime=1731971837382; duration=0sec 2024-11-18T23:17:17,466 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:17,466 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:B 2024-11-18T23:17:17,467 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:17,467 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/C is initiating minor compaction (all files) 2024-11-18T23:17:17,468 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/C in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:17,468 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bb1feb17120f472d8d96ed36d435c50c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e8cae4cc22e445ca93c11d7aa23d9824, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/24d4f5f1b5144f8d8b553e36c7bf92d8] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=35.3 K 2024-11-18T23:17:17,469 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1feb17120f472d8d96ed36d435c50c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731971831541 2024-11-18T23:17:17,469 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8cae4cc22e445ca93c11d7aa23d9824, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731971833894 2024-11-18T23:17:17,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/cb829408f7b9412b8ee6985fe1d9c08b is 50, key is test_row_0/A:col10/1731971836456/Put/seqid=0 2024-11-18T23:17:17,471 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d4f5f1b5144f8d8b553e36c7bf92d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731971835177 2024-11-18T23:17:17,482 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:17,483 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/03284bb54b8a4b98aec302eb48367eeb is 50, key is test_row_0/C:col10/1731971835177/Put/seqid=0 2024-11-18T23:17:17,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742267_1443 (size=12104) 2024-11-18T23:17:17,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742266_1442 (size=12001) 2024-11-18T23:17:17,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:17,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:17,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971897684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971897686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971897692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,704 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971897701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971897706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971897807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971897807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971897807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971897818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:17,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971897827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:17,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:17,920 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/cb829408f7b9412b8ee6985fe1d9c08b 2024-11-18T23:17:17,940 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/03284bb54b8a4b98aec302eb48367eeb as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/03284bb54b8a4b98aec302eb48367eeb 2024-11-18T23:17:17,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/3efa518187bf4be6ad823b533e7987f6 is 50, key is test_row_0/B:col10/1731971836456/Put/seqid=0 2024-11-18T23:17:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T23:17:17,968 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/C of 37a0e116df39ce62ed45cb517bbc0fb6 into 03284bb54b8a4b98aec302eb48367eeb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:17,968 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:17,968 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/C, priority=13, startTime=1731971837383; duration=0sec 2024-11-18T23:17:17,968 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:17,968 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:C 2024-11-18T23:17:17,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742268_1444 (size=12001) 2024-11-18T23:17:18,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971898018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971898018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971898027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971898038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971898048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971898339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971898339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971898348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971898348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971898371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,391 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/3efa518187bf4be6ad823b533e7987f6 2024-11-18T23:17:18,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/cfd8c26570014e888f0b702f9a1b5863 is 50, key is test_row_0/C:col10/1731971836456/Put/seqid=0 2024-11-18T23:17:18,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742269_1445 (size=12001) 2024-11-18T23:17:18,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971898857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971898857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,876 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/cfd8c26570014e888f0b702f9a1b5863 2024-11-18T23:17:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971898869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971898870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/cb829408f7b9412b8ee6985fe1d9c08b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/cb829408f7b9412b8ee6985fe1d9c08b 2024-11-18T23:17:18,891 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/cb829408f7b9412b8ee6985fe1d9c08b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T23:17:18,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/3efa518187bf4be6ad823b533e7987f6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/3efa518187bf4be6ad823b533e7987f6 2024-11-18T23:17:18,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971898887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:18,896 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/3efa518187bf4be6ad823b533e7987f6, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T23:17:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:18,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:18,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/cfd8c26570014e888f0b702f9a1b5863 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/cfd8c26570014e888f0b702f9a1b5863 2024-11-18T23:17:18,915 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/cfd8c26570014e888f0b702f9a1b5863, entries=150, sequenceid=116, filesize=11.7 K 2024-11-18T23:17:18,917 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1464ms, sequenceid=116, compaction requested=false 2024-11-18T23:17:18,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:18,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:18,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-18T23:17:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-18T23:17:18,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-18T23:17:18,924 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0890 sec 2024-11-18T23:17:18,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.0960 sec 2024-11-18T23:17:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-18T23:17:18,991 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:18,992 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-18T23:17:18,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:18,995 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:18,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T23:17:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T23:17:19,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-18T23:17:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:19,151 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:17:19,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:19,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/185740bc6d6243bbb4c912ecffb2f33e is 50, key is test_row_0/A:col10/1731971837700/Put/seqid=0 2024-11-18T23:17:19,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742270_1446 (size=12051) 2024-11-18T23:17:19,197 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/185740bc6d6243bbb4c912ecffb2f33e 2024-11-18T23:17:19,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/749d1a059f514075bd7eac43b1ecd901 is 50, key is test_row_0/B:col10/1731971837700/Put/seqid=0 2024-11-18T23:17:19,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742271_1447 (size=12051) 2024-11-18T23:17:19,275 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/749d1a059f514075bd7eac43b1ecd901 2024-11-18T23:17:19,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/2f57b1faf76a4611b394ef735d8bdf24 is 50, key is test_row_0/C:col10/1731971837700/Put/seqid=0 2024-11-18T23:17:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T23:17:19,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742272_1448 (size=12051) 2024-11-18T23:17:19,374 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/2f57b1faf76a4611b394ef735d8bdf24 2024-11-18T23:17:19,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/185740bc6d6243bbb4c912ecffb2f33e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/185740bc6d6243bbb4c912ecffb2f33e 2024-11-18T23:17:19,393 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/185740bc6d6243bbb4c912ecffb2f33e, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T23:17:19,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/749d1a059f514075bd7eac43b1ecd901 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/749d1a059f514075bd7eac43b1ecd901 2024-11-18T23:17:19,412 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/749d1a059f514075bd7eac43b1ecd901, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T23:17:19,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/2f57b1faf76a4611b394ef735d8bdf24 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2f57b1faf76a4611b394ef735d8bdf24 2024-11-18T23:17:19,420 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2f57b1faf76a4611b394ef735d8bdf24, entries=150, sequenceid=132, filesize=11.8 K 2024-11-18T23:17:19,421 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 37a0e116df39ce62ed45cb517bbc0fb6 in 270ms, sequenceid=132, compaction requested=true 2024-11-18T23:17:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:19,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-18T23:17:19,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-18T23:17:19,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-18T23:17:19,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 428 msec 2024-11-18T23:17:19,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 434 msec 2024-11-18T23:17:19,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-18T23:17:19,633 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:19,633 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:19,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-18T23:17:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T23:17:19,644 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:19,645 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:19,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T23:17:19,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-18T23:17:19,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:19,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-18T23:17:19,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-18T23:17:19,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-18T23:17:19,820 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 163 msec 2024-11-18T23:17:19,852 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 191 msec 2024-11-18T23:17:19,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:19,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:17:19,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:19,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:19,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:19,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:19,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:19,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-18T23:17:19,958 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:19,959 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:19,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-18T23:17:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T23:17:19,965 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:19,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fdf4da3064494ac59f9f9c860f7e1c74 is 50, key is test_row_0/A:col10/1731971839893/Put/seqid=0 2024-11-18T23:17:19,969 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:19,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:20,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742273_1449 (size=12151) 2024-11-18T23:17:20,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fdf4da3064494ac59f9f9c860f7e1c74 2024-11-18T23:17:20,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T23:17:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971900062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971900062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/4f8c68f58534418fb289a1eab7b859b3 is 50, key is test_row_0/B:col10/1731971839893/Put/seqid=0 2024-11-18T23:17:20,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971900067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,100 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971900069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971900072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742274_1450 (size=12151) 2024-11-18T23:17:20,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/4f8c68f58534418fb289a1eab7b859b3 2024-11-18T23:17:20,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/65217e9f662d445d998c9ba308c9a53c is 50, key is test_row_0/C:col10/1731971839893/Put/seqid=0 2024-11-18T23:17:20,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971900184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971900188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971900208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742275_1451 (size=12151) 2024-11-18T23:17:20,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/65217e9f662d445d998c9ba308c9a53c 2024-11-18T23:17:20,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971900222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971900222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,253 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fdf4da3064494ac59f9f9c860f7e1c74 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fdf4da3064494ac59f9f9c860f7e1c74 2024-11-18T23:17:20,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T23:17:20,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fdf4da3064494ac59f9f9c860f7e1c74, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T23:17:20,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/4f8c68f58534418fb289a1eab7b859b3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/4f8c68f58534418fb289a1eab7b859b3 2024-11-18T23:17:20,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/4f8c68f58534418fb289a1eab7b859b3, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T23:17:20,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/65217e9f662d445d998c9ba308c9a53c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/65217e9f662d445d998c9ba308c9a53c 2024-11-18T23:17:20,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/65217e9f662d445d998c9ba308c9a53c, entries=150, sequenceid=145, filesize=11.9 K 2024-11-18T23:17:20,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 37a0e116df39ce62ed45cb517bbc0fb6 in 401ms, sequenceid=145, compaction requested=true 2024-11-18T23:17:20,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:20,337 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:20,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:20,337 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:20,345 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48307 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:20,345 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/A is initiating minor compaction (all files) 2024-11-18T23:17:20,345 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/A in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,345 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/6c71ce65064b41f7ae841e78669c6059, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/cb829408f7b9412b8ee6985fe1d9c08b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/185740bc6d6243bbb4c912ecffb2f33e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fdf4da3064494ac59f9f9c860f7e1c74] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=47.2 K 2024-11-18T23:17:20,352 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48307 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:20,352 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/B is initiating minor compaction (all files) 2024-11-18T23:17:20,353 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/B in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,353 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/92dd4d6e288d4bc7af0d11c288310701, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/3efa518187bf4be6ad823b533e7987f6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/749d1a059f514075bd7eac43b1ecd901, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/4f8c68f58534418fb289a1eab7b859b3] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=47.2 K 2024-11-18T23:17:20,353 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c71ce65064b41f7ae841e78669c6059, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731971835177 2024-11-18T23:17:20,357 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 92dd4d6e288d4bc7af0d11c288310701, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731971835177 2024-11-18T23:17:20,357 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb829408f7b9412b8ee6985fe1d9c08b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731971836441 2024-11-18T23:17:20,357 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3efa518187bf4be6ad823b533e7987f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731971836441 2024-11-18T23:17:20,357 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185740bc6d6243bbb4c912ecffb2f33e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731971837677 2024-11-18T23:17:20,363 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 749d1a059f514075bd7eac43b1ecd901, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731971837677 2024-11-18T23:17:20,363 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdf4da3064494ac59f9f9c860f7e1c74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971839893 2024-11-18T23:17:20,364 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f8c68f58534418fb289a1eab7b859b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971839893 2024-11-18T23:17:20,407 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#B#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:20,408 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/0e20b014821b4d61b228baf363ff2e16 is 50, key is test_row_0/B:col10/1731971839893/Put/seqid=0 2024-11-18T23:17:20,434 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#A#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:20,435 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/4a0ca9bde52c4546a2f1c5fa44e9d00f is 50, key is test_row_0/A:col10/1731971839893/Put/seqid=0 2024-11-18T23:17:20,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-18T23:17:20,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971840062, max timestamp=1731971840408Num uniques 859; 2024-11-18T23:17:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:20,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:20,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:20,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:20,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:20,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:20,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:20,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:20,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:20,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/3d21124378f645758d4cecc6cc032ec3 is 50, key is test_row_0/A:col10/1731971840070/Put/seqid=0 2024-11-18T23:17:20,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742276_1452 (size=12391) 2024-11-18T23:17:20,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742278_1454 (size=14541) 2024-11-18T23:17:20,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742277_1453 (size=12391) 2024-11-18T23:17:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971900520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971900522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T23:17:20,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971900581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971900583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971900583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971900688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971900692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971900720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971900736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971900737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,901 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/0e20b014821b4d61b228baf363ff2e16 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/0e20b014821b4d61b228baf363ff2e16 2024-11-18T23:17:20,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/3d21124378f645758d4cecc6cc032ec3 2024-11-18T23:17:20,909 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/B of 37a0e116df39ce62ed45cb517bbc0fb6 into 0e20b014821b4d61b228baf363ff2e16(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:20,909 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:20,909 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/B, priority=12, startTime=1731971840337; duration=0sec 2024-11-18T23:17:20,909 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:20,909 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:B 2024-11-18T23:17:20,909 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:20,911 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48307 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:20,911 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/C is initiating minor compaction (all files) 2024-11-18T23:17:20,911 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/C in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,911 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/03284bb54b8a4b98aec302eb48367eeb, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/cfd8c26570014e888f0b702f9a1b5863, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2f57b1faf76a4611b394ef735d8bdf24, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/65217e9f662d445d998c9ba308c9a53c] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=47.2 K 2024-11-18T23:17:20,919 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 03284bb54b8a4b98aec302eb48367eeb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731971835177 2024-11-18T23:17:20,919 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting cfd8c26570014e888f0b702f9a1b5863, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731971836441 2024-11-18T23:17:20,920 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f57b1faf76a4611b394ef735d8bdf24, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731971837677 2024-11-18T23:17:20,920 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 65217e9f662d445d998c9ba308c9a53c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971839893 2024-11-18T23:17:20,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7225a7b7284f4a90b927995545eb1328 is 50, key is test_row_0/B:col10/1731971840070/Put/seqid=0 2024-11-18T23:17:20,944 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#C#compaction#384 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:20,945 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/6068b0df05bc414398e00a86034d7b3e is 50, key is test_row_0/C:col10/1731971839893/Put/seqid=0 2024-11-18T23:17:20,964 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/4a0ca9bde52c4546a2f1c5fa44e9d00f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a0ca9bde52c4546a2f1c5fa44e9d00f 2024-11-18T23:17:20,970 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/A of 37a0e116df39ce62ed45cb517bbc0fb6 into 4a0ca9bde52c4546a2f1c5fa44e9d00f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:20,970 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:20,970 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/A, priority=12, startTime=1731971840337; duration=0sec 2024-11-18T23:17:20,970 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:20,970 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:A 2024-11-18T23:17:20,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:20,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:20,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:20,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971900948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971900948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971900967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971900972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:20,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:20,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742279_1455 (size=12151) 2024-11-18T23:17:21,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7225a7b7284f4a90b927995545eb1328 2024-11-18T23:17:21,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971900979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742280_1456 (size=12391) 2024-11-18T23:17:21,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/d0417003a9134e3f82f7e7a70b6422df is 50, key is test_row_0/C:col10/1731971840070/Put/seqid=0 2024-11-18T23:17:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T23:17:21,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742281_1457 (size=12151) 2024-11-18T23:17:21,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:21,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:21,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971901287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971901290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:21,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971901297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971901300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971901328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:21,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:21,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:21,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/d0417003a9134e3f82f7e7a70b6422df 2024-11-18T23:17:21,538 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/6068b0df05bc414398e00a86034d7b3e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6068b0df05bc414398e00a86034d7b3e 2024-11-18T23:17:21,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/3d21124378f645758d4cecc6cc032ec3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/3d21124378f645758d4cecc6cc032ec3 2024-11-18T23:17:21,549 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/C of 37a0e116df39ce62ed45cb517bbc0fb6 into 6068b0df05bc414398e00a86034d7b3e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:21,549 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:21,549 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/C, priority=12, startTime=1731971840337; duration=0sec 2024-11-18T23:17:21,549 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:21,549 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:C 2024-11-18T23:17:21,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/3d21124378f645758d4cecc6cc032ec3, entries=200, sequenceid=166, filesize=14.2 K 2024-11-18T23:17:21,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7225a7b7284f4a90b927995545eb1328 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7225a7b7284f4a90b927995545eb1328 2024-11-18T23:17:21,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7225a7b7284f4a90b927995545eb1328, entries=150, sequenceid=166, filesize=11.9 K 2024-11-18T23:17:21,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/d0417003a9134e3f82f7e7a70b6422df as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d0417003a9134e3f82f7e7a70b6422df 2024-11-18T23:17:21,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d0417003a9134e3f82f7e7a70b6422df, entries=150, sequenceid=166, filesize=11.9 K 2024-11-18T23:17:21,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1135ms, sequenceid=166, compaction requested=false 2024-11-18T23:17:21,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:21,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-18T23:17:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:21,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:17:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:21,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:21,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:21,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fb8b83407b2a42c79766d72db4f0d675 is 50, key is test_row_0/A:col10/1731971840575/Put/seqid=0 2024-11-18T23:17:21,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742282_1458 (size=12151) 2024-11-18T23:17:21,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:21,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:21,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971901925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971901925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971901928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,972 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971901932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:21,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971901933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:21,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971902077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971902078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971902078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971902081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,113 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T23:17:22,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971902087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,136 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fb8b83407b2a42c79766d72db4f0d675 2024-11-18T23:17:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a252021a061e4237ac66c77177d9832e is 50, key is test_row_0/B:col10/1731971840575/Put/seqid=0 2024-11-18T23:17:22,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742283_1459 (size=12151) 2024-11-18T23:17:22,238 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a252021a061e4237ac66c77177d9832e 2024-11-18T23:17:22,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/08aedd1533a34091917d00ff2ce3f8fc is 50, key is test_row_0/C:col10/1731971840575/Put/seqid=0 2024-11-18T23:17:22,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742284_1460 (size=12151) 2024-11-18T23:17:22,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971902317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,345 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/08aedd1533a34091917d00ff2ce3f8fc 2024-11-18T23:17:22,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971902317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971902318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,358 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971902318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fb8b83407b2a42c79766d72db4f0d675 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fb8b83407b2a42c79766d72db4f0d675 2024-11-18T23:17:22,390 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fb8b83407b2a42c79766d72db4f0d675, entries=150, sequenceid=185, filesize=11.9 K 2024-11-18T23:17:22,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971902356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a252021a061e4237ac66c77177d9832e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a252021a061e4237ac66c77177d9832e 2024-11-18T23:17:22,410 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a252021a061e4237ac66c77177d9832e, entries=150, sequenceid=185, filesize=11.9 K 2024-11-18T23:17:22,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/08aedd1533a34091917d00ff2ce3f8fc as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/08aedd1533a34091917d00ff2ce3f8fc 2024-11-18T23:17:22,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/08aedd1533a34091917d00ff2ce3f8fc, entries=150, sequenceid=185, filesize=11.9 K 2024-11-18T23:17:22,455 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 37a0e116df39ce62ed45cb517bbc0fb6 in 810ms, sequenceid=185, compaction requested=true 2024-11-18T23:17:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:22,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-18T23:17:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-18T23:17:22,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-18T23:17:22,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4910 sec 2024-11-18T23:17:22,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.5120 sec 2024-11-18T23:17:22,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-18T23:17:22,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:22,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:22,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:22,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:22,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:22,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:22,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:22,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/24018519cde14f5a9b2fd53702063168 is 50, key is test_row_0/A:col10/1731971841928/Put/seqid=0 2024-11-18T23:17:22,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742285_1461 (size=14541) 2024-11-18T23:17:22,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971902718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971902729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971902731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971902738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971902742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,781 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971902881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971902881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971902881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971902888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971902888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,922 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:22,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/24018519cde14f5a9b2fd53702063168 2024-11-18T23:17:23,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971903128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971903128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971903129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971903129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971903129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e89d107e499f4538a437fb74c87323f4 is 50, key is test_row_0/B:col10/1731971841928/Put/seqid=0 2024-11-18T23:17:23,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742286_1462 (size=12151) 2024-11-18T23:17:23,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e89d107e499f4538a437fb74c87323f4 2024-11-18T23:17:23,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/4e0c406fe1574cd687737f1372d71b43 is 50, key is test_row_0/C:col10/1731971841928/Put/seqid=0 2024-11-18T23:17:23,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742287_1463 (size=12151) 2024-11-18T23:17:23,464 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971903461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971903461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971903461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971903461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971903464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,467 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:23,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/4e0c406fe1574cd687737f1372d71b43 2024-11-18T23:17:23,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/24018519cde14f5a9b2fd53702063168 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/24018519cde14f5a9b2fd53702063168 2024-11-18T23:17:23,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/24018519cde14f5a9b2fd53702063168, entries=200, sequenceid=202, filesize=14.2 K 2024-11-18T23:17:23,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e89d107e499f4538a437fb74c87323f4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e89d107e499f4538a437fb74c87323f4 2024-11-18T23:17:23,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e89d107e499f4538a437fb74c87323f4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-18T23:17:23,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/4e0c406fe1574cd687737f1372d71b43 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/4e0c406fe1574cd687737f1372d71b43 2024-11-18T23:17:23,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/4e0c406fe1574cd687737f1372d71b43, entries=150, sequenceid=202, filesize=11.9 K 2024-11-18T23:17:23,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1106ms, sequenceid=202, compaction requested=true 2024-11-18T23:17:23,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:23,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:23,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:23,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:23,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:23,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:23,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T23:17:23,764 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:23,764 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:23,780 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:23,780 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/C is initiating minor compaction (all files) 2024-11-18T23:17:23,780 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/C in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:23,781 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6068b0df05bc414398e00a86034d7b3e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d0417003a9134e3f82f7e7a70b6422df, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/08aedd1533a34091917d00ff2ce3f8fc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/4e0c406fe1574cd687737f1372d71b43] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=47.7 K 2024-11-18T23:17:23,784 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53624 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:23,784 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/A is initiating minor compaction (all files) 2024-11-18T23:17:23,784 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/A in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:23,784 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a0ca9bde52c4546a2f1c5fa44e9d00f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/3d21124378f645758d4cecc6cc032ec3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fb8b83407b2a42c79766d72db4f0d675, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/24018519cde14f5a9b2fd53702063168] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=52.4 K 2024-11-18T23:17:23,785 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6068b0df05bc414398e00a86034d7b3e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971839893 2024-11-18T23:17:23,788 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a0ca9bde52c4546a2f1c5fa44e9d00f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971839893 2024-11-18T23:17:23,789 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d0417003a9134e3f82f7e7a70b6422df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731971840067 2024-11-18T23:17:23,794 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d21124378f645758d4cecc6cc032ec3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731971840062 2024-11-18T23:17:23,795 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 08aedd1533a34091917d00ff2ce3f8fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731971840514 2024-11-18T23:17:23,797 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb8b83407b2a42c79766d72db4f0d675, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731971840514 2024-11-18T23:17:23,797 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e0c406fe1574cd687737f1372d71b43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731971841882 2024-11-18T23:17:23,804 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24018519cde14f5a9b2fd53702063168, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731971841882 2024-11-18T23:17:23,843 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:23,844 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/6e79fe0eeac848d3b0c29f551704b601 is 50, key is test_row_0/C:col10/1731971841928/Put/seqid=0 2024-11-18T23:17:23,856 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#A#compaction#393 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:23,856 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/1a1e5719030e4d21ab2159bf7c1986da is 50, key is test_row_0/A:col10/1731971841928/Put/seqid=0 2024-11-18T23:17:23,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742288_1464 (size=12527) 2024-11-18T23:17:23,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742289_1465 (size=12527) 2024-11-18T23:17:23,911 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/6e79fe0eeac848d3b0c29f551704b601 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6e79fe0eeac848d3b0c29f551704b601 2024-11-18T23:17:23,912 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/1a1e5719030e4d21ab2159bf7c1986da as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1a1e5719030e4d21ab2159bf7c1986da 2024-11-18T23:17:23,917 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/C of 37a0e116df39ce62ed45cb517bbc0fb6 into 6e79fe0eeac848d3b0c29f551704b601(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:23,917 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:23,917 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/C, priority=12, startTime=1731971843763; duration=0sec 2024-11-18T23:17:23,917 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:23,917 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:C 2024-11-18T23:17:23,917 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:23,918 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:23,918 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/B is initiating minor compaction (all files) 2024-11-18T23:17:23,918 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/B in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:23,918 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/0e20b014821b4d61b228baf363ff2e16, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7225a7b7284f4a90b927995545eb1328, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a252021a061e4237ac66c77177d9832e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e89d107e499f4538a437fb74c87323f4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=47.7 K 2024-11-18T23:17:23,918 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e20b014821b4d61b228baf363ff2e16, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971839893 2024-11-18T23:17:23,919 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 7225a7b7284f4a90b927995545eb1328, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731971840067 2024-11-18T23:17:23,919 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a252021a061e4237ac66c77177d9832e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731971840514 2024-11-18T23:17:23,919 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e89d107e499f4538a437fb74c87323f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731971841882 2024-11-18T23:17:23,925 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/A of 37a0e116df39ce62ed45cb517bbc0fb6 into 1a1e5719030e4d21ab2159bf7c1986da(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:23,925 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:23,925 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/A, priority=12, startTime=1731971843762; duration=0sec 2024-11-18T23:17:23,925 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:23,925 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:A 2024-11-18T23:17:23,936 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#B#compaction#394 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:23,937 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/bbb8612a88314cf08d824da186286a3a is 50, key is test_row_0/B:col10/1731971841928/Put/seqid=0 2024-11-18T23:17:23,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742290_1466 (size=12527) 2024-11-18T23:17:23,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:17:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:23,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:24,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/83d5cc1787fb40f193ec1bd26e7e8248 is 50, key is test_row_0/A:col10/1731971842725/Put/seqid=0 2024-11-18T23:17:24,037 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/bbb8612a88314cf08d824da186286a3a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bbb8612a88314cf08d824da186286a3a 2024-11-18T23:17:24,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971904047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,062 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/B of 37a0e116df39ce62ed45cb517bbc0fb6 into bbb8612a88314cf08d824da186286a3a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:24,062 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:24,062 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/B, priority=12, startTime=1731971843763; duration=0sec 2024-11-18T23:17:24,062 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:24,062 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:B 2024-11-18T23:17:24,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742291_1467 (size=14541) 2024-11-18T23:17:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971904055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971904058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971904062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971904062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-18T23:17:24,128 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:24,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:24,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-18T23:17:24,134 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T23:17:24,135 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:24,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:24,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971904167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971904189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971904198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971904198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971904200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T23:17:24,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971904378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971904407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971904423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971904423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971904429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T23:17:24,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:24,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:24,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/83d5cc1787fb40f193ec1bd26e7e8248 2024-11-18T23:17:24,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a3259e8eda9941f2858e6056cd0062f4 is 50, key is test_row_0/B:col10/1731971842725/Put/seqid=0 2024-11-18T23:17:24,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742292_1468 (size=12151) 2024-11-18T23:17:24,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971904700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971904729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971904747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971904747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T23:17:24,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971904747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:24,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:24,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:24,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:24,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:24,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a3259e8eda9941f2858e6056cd0062f4 2024-11-18T23:17:25,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/5cd8ac1051fc46bc9f4dfe50d10c5b77 is 50, key is test_row_0/C:col10/1731971842725/Put/seqid=0 2024-11-18T23:17:25,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742293_1469 (size=12151) 2024-11-18T23:17:25,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:25,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:25,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971905229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:25,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T23:17:25,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:25,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971905257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971905268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971905269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971905269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:25,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:25,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:25,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:25,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:25,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:25,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/5cd8ac1051fc46bc9f4dfe50d10c5b77 2024-11-18T23:17:25,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/83d5cc1787fb40f193ec1bd26e7e8248 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/83d5cc1787fb40f193ec1bd26e7e8248 2024-11-18T23:17:25,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/83d5cc1787fb40f193ec1bd26e7e8248, entries=200, sequenceid=220, filesize=14.2 K 2024-11-18T23:17:25,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a3259e8eda9941f2858e6056cd0062f4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a3259e8eda9941f2858e6056cd0062f4 2024-11-18T23:17:25,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a3259e8eda9941f2858e6056cd0062f4, entries=150, sequenceid=220, filesize=11.9 K 2024-11-18T23:17:25,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/5cd8ac1051fc46bc9f4dfe50d10c5b77 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/5cd8ac1051fc46bc9f4dfe50d10c5b77 2024-11-18T23:17:25,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/5cd8ac1051fc46bc9f4dfe50d10c5b77, entries=150, sequenceid=220, filesize=11.9 K 2024-11-18T23:17:25,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1565ms, sequenceid=220, compaction requested=false 2024-11-18T23:17:25,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:25,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-18T23:17:25,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-18T23:17:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:25,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:25,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/912d2a109dda4f239be064edbf0ec736 is 50, key is test_row_0/A:col10/1731971844042/Put/seqid=0 2024-11-18T23:17:25,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742294_1470 (size=12151) 2024-11-18T23:17:25,713 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/912d2a109dda4f239be064edbf0ec736 2024-11-18T23:17:25,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7add1a88155d438b829651dcfe21b158 is 50, key is test_row_0/B:col10/1731971844042/Put/seqid=0 2024-11-18T23:17:25,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742295_1471 (size=12151) 2024-11-18T23:17:25,769 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7add1a88155d438b829651dcfe21b158 2024-11-18T23:17:25,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/acf323ee0a8447f5871bcf762f17a037 is 50, key is test_row_0/C:col10/1731971844042/Put/seqid=0 2024-11-18T23:17:25,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742296_1472 (size=12151) 2024-11-18T23:17:25,828 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/acf323ee0a8447f5871bcf762f17a037 2024-11-18T23:17:25,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/912d2a109dda4f239be064edbf0ec736 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/912d2a109dda4f239be064edbf0ec736 2024-11-18T23:17:25,887 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/912d2a109dda4f239be064edbf0ec736, entries=150, sequenceid=237, filesize=11.9 K 2024-11-18T23:17:25,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7add1a88155d438b829651dcfe21b158 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7add1a88155d438b829651dcfe21b158 2024-11-18T23:17:25,903 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7add1a88155d438b829651dcfe21b158, entries=150, sequenceid=237, filesize=11.9 K 2024-11-18T23:17:25,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/acf323ee0a8447f5871bcf762f17a037 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/acf323ee0a8447f5871bcf762f17a037 2024-11-18T23:17:25,911 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/acf323ee0a8447f5871bcf762f17a037, entries=150, sequenceid=237, filesize=11.9 K 2024-11-18T23:17:25,913 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=0 B/0 for 37a0e116df39ce62ed45cb517bbc0fb6 in 272ms, sequenceid=237, compaction requested=true 2024-11-18T23:17:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:25,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-18T23:17:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-18T23:17:25,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-18T23:17:25,919 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-18T23:17:25,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7920 sec 2024-11-18T23:17:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-18T23:17:26,283 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:26,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-18T23:17:26,287 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T23:17:26,289 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:26,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:26,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:17:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:26,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/1d02479e090d44dea029df3618657772 is 50, key is test_row_0/A:col10/1731971846295/Put/seqid=0 2024-11-18T23:17:26,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742297_1473 (size=14541) 2024-11-18T23:17:26,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/1d02479e090d44dea029df3618657772 2024-11-18T23:17:26,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/1574b8b319864fc89dde5627ba3c13a6 is 50, key is test_row_0/B:col10/1731971846295/Put/seqid=0 2024-11-18T23:17:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T23:17:26,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971846363, max timestamp=1731971846389Num uniques 859; 2024-11-18T23:17:26,400 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:26,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971846363, max timestamp=1731971846389Num uniques 859; 2024-11-18T23:17:26,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:26,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971846363, max timestamp=1731971846389Num uniques 859; 2024-11-18T23:17:26,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:26,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742298_1474 (size=12151) 2024-11-18T23:17:26,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/1574b8b319864fc89dde5627ba3c13a6 2024-11-18T23:17:26,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T23:17:26,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:26,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/d7888d6119d14536a72f6f7885c11e50 is 50, key is test_row_0/C:col10/1731971846295/Put/seqid=0 2024-11-18T23:17:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971906472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971906501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971906504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971906506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971906506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742299_1475 (size=12151) 2024-11-18T23:17:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T23:17:26,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T23:17:26,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971906617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971906620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971906638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971906638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971906638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,651 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T23:17:26,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:26,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971906848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971906849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971906860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971906860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971906860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:26,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T23:17:26,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T23:17:26,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:26,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:26,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:26,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/d7888d6119d14536a72f6f7885c11e50 2024-11-18T23:17:26,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/1d02479e090d44dea029df3618657772 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1d02479e090d44dea029df3618657772 2024-11-18T23:17:26,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1d02479e090d44dea029df3618657772, entries=200, sequenceid=247, filesize=14.2 K 2024-11-18T23:17:26,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/1574b8b319864fc89dde5627ba3c13a6 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/1574b8b319864fc89dde5627ba3c13a6 2024-11-18T23:17:27,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/1574b8b319864fc89dde5627ba3c13a6, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T23:17:27,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/d7888d6119d14536a72f6f7885c11e50 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d7888d6119d14536a72f6f7885c11e50 2024-11-18T23:17:27,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d7888d6119d14536a72f6f7885c11e50, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T23:17:27,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 37a0e116df39ce62ed45cb517bbc0fb6 in 716ms, sequenceid=247, compaction requested=true 2024-11-18T23:17:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:27,012 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:27,012 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:27,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:27,013 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:27,013 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:27,013 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/A is initiating minor compaction (all files) 2024-11-18T23:17:27,013 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/B is initiating minor compaction (all files) 2024-11-18T23:17:27,013 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/A in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:27,013 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/B in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:27,013 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1a1e5719030e4d21ab2159bf7c1986da, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/83d5cc1787fb40f193ec1bd26e7e8248, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/912d2a109dda4f239be064edbf0ec736, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1d02479e090d44dea029df3618657772] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=52.5 K 2024-11-18T23:17:27,013 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bbb8612a88314cf08d824da186286a3a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a3259e8eda9941f2858e6056cd0062f4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7add1a88155d438b829651dcfe21b158, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/1574b8b319864fc89dde5627ba3c13a6] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=47.8 K 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting bbb8612a88314cf08d824da186286a3a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731971841882 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1e5719030e4d21ab2159bf7c1986da, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731971841882 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a3259e8eda9941f2858e6056cd0062f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731971842725 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83d5cc1787fb40f193ec1bd26e7e8248, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731971842725 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 7add1a88155d438b829651dcfe21b158, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731971844019 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 912d2a109dda4f239be064edbf0ec736, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731971844019 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 1574b8b319864fc89dde5627ba3c13a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971846286 2024-11-18T23:17:27,014 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d02479e090d44dea029df3618657772, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971846286 2024-11-18T23:17:27,023 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#B#compaction#404 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:27,024 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/5ac47073e7f440d18908fc3bd1990fe2 is 50, key is test_row_0/B:col10/1731971846295/Put/seqid=0 2024-11-18T23:17:27,030 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#A#compaction#405 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:27,031 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/46ddc2f4cccf4e4887815ab3ee470fe5 is 50, key is test_row_0/A:col10/1731971846295/Put/seqid=0 2024-11-18T23:17:27,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742300_1476 (size=12663) 2024-11-18T23:17:27,045 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/5ac47073e7f440d18908fc3bd1990fe2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/5ac47073e7f440d18908fc3bd1990fe2 2024-11-18T23:17:27,053 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/B of 37a0e116df39ce62ed45cb517bbc0fb6 into 5ac47073e7f440d18908fc3bd1990fe2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:27,053 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:27,053 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/B, priority=12, startTime=1731971847012; duration=0sec 2024-11-18T23:17:27,053 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:27,053 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:B 2024-11-18T23:17:27,054 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:27,062 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:27,062 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/C is initiating minor compaction (all files) 2024-11-18T23:17:27,062 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/C in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:27,062 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6e79fe0eeac848d3b0c29f551704b601, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/5cd8ac1051fc46bc9f4dfe50d10c5b77, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/acf323ee0a8447f5871bcf762f17a037, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d7888d6119d14536a72f6f7885c11e50] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=47.8 K 2024-11-18T23:17:27,068 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e79fe0eeac848d3b0c29f551704b601, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731971841882 2024-11-18T23:17:27,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742301_1477 (size=12663) 2024-11-18T23:17:27,069 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd8ac1051fc46bc9f4dfe50d10c5b77, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731971842725 2024-11-18T23:17:27,071 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting acf323ee0a8447f5871bcf762f17a037, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731971844019 2024-11-18T23:17:27,073 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting d7888d6119d14536a72f6f7885c11e50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971846286 2024-11-18T23:17:27,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-18T23:17:27,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:27,079 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-18T23:17:27,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:27,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:27,088 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/46ddc2f4cccf4e4887815ab3ee470fe5 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/46ddc2f4cccf4e4887815ab3ee470fe5 2024-11-18T23:17:27,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/732b54ce23784be19119d362200c1a33 is 50, key is test_row_0/A:col10/1731971846498/Put/seqid=0 2024-11-18T23:17:27,098 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/A of 37a0e116df39ce62ed45cb517bbc0fb6 into 46ddc2f4cccf4e4887815ab3ee470fe5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:27,098 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:27,098 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/A, priority=12, startTime=1731971847012; duration=0sec 2024-11-18T23:17:27,098 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:27,098 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:A 2024-11-18T23:17:27,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742302_1478 (size=12301) 2024-11-18T23:17:27,117 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/732b54ce23784be19119d362200c1a33 2024-11-18T23:17:27,125 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#C#compaction#407 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:27,125 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/e259a97e25f1495cb624c321a715747b is 50, key is test_row_0/C:col10/1731971846295/Put/seqid=0 2024-11-18T23:17:27,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/f78199bff8164465be3d6cb4ef630c78 is 50, key is test_row_0/B:col10/1731971846498/Put/seqid=0 2024-11-18T23:17:27,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742303_1479 (size=12663) 2024-11-18T23:17:27,174 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/e259a97e25f1495cb624c321a715747b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e259a97e25f1495cb624c321a715747b 2024-11-18T23:17:27,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742304_1480 (size=12301) 2024-11-18T23:17:27,182 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/C of 37a0e116df39ce62ed45cb517bbc0fb6 into e259a97e25f1495cb624c321a715747b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:27,182 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:27,182 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/C, priority=12, startTime=1731971847012; duration=0sec 2024-11-18T23:17:27,182 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:27,182 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:C 2024-11-18T23:17:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:27,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:27,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971907194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971907195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971907197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971907200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971907200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971907307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971907317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971907318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971907318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971907320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T23:17:27,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971907521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971907541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971907541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971907550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971907549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,577 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/f78199bff8164465be3d6cb4ef630c78 2024-11-18T23:17:27,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/af71db8476344f56bb4298751a38abce is 50, key is test_row_0/C:col10/1731971846498/Put/seqid=0 2024-11-18T23:17:27,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742305_1481 (size=12301) 2024-11-18T23:17:27,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971907840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971907864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971907868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971907888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971907888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,091 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/af71db8476344f56bb4298751a38abce 2024-11-18T23:17:28,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/732b54ce23784be19119d362200c1a33 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/732b54ce23784be19119d362200c1a33 2024-11-18T23:17:28,104 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/732b54ce23784be19119d362200c1a33, entries=150, sequenceid=285, filesize=12.0 K 2024-11-18T23:17:28,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/f78199bff8164465be3d6cb4ef630c78 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/f78199bff8164465be3d6cb4ef630c78 2024-11-18T23:17:28,124 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/f78199bff8164465be3d6cb4ef630c78, entries=150, sequenceid=285, filesize=12.0 K 2024-11-18T23:17:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/af71db8476344f56bb4298751a38abce as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/af71db8476344f56bb4298751a38abce 2024-11-18T23:17:28,143 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/af71db8476344f56bb4298751a38abce, entries=150, sequenceid=285, filesize=12.0 K 2024-11-18T23:17:28,145 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1066ms, sequenceid=285, compaction requested=false 2024-11-18T23:17:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:28,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-18T23:17:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-18T23:17:28,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-18T23:17:28,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8680 sec 2024-11-18T23:17:28,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8840 sec 2024-11-18T23:17:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:28,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:17:28,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:28,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:28,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:28,425 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/a5448f94a8a04399b64bfb6ec20fbb2d is 50, key is test_row_0/A:col10/1731971847196/Put/seqid=0 2024-11-18T23:17:28,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-18T23:17:28,449 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:28,453 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:28,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-18T23:17:28,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742306_1482 (size=14741) 2024-11-18T23:17:28,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/a5448f94a8a04399b64bfb6ec20fbb2d 2024-11-18T23:17:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T23:17:28,461 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:28,463 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:28,463 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:28,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/bb5fe63c791c4e69a61e3907fc7770a4 is 50, key is test_row_0/B:col10/1731971847196/Put/seqid=0 2024-11-18T23:17:28,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971908489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731971848474, max timestamp=1731971848510Num uniques 859; 2024-11-18T23:17:28,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731971848474, max timestamp=1731971848510Num uniques 859; 2024-11-18T23:17:28,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T23:17:28,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T23:17:28,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731971848474, max timestamp=1731971848510Num uniques 859; 2024-11-18T23:17:28,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-18T23:17:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742307_1483 (size=12301) 2024-11-18T23:17:28,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/bb5fe63c791c4e69a61e3907fc7770a4 2024-11-18T23:17:28,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T23:17:28,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/220c9d1fb9c242fdb5ae351fb986228d is 50, key is test_row_0/C:col10/1731971847196/Put/seqid=0 2024-11-18T23:17:28,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742308_1484 (size=12301) 2024-11-18T23:17:28,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/220c9d1fb9c242fdb5ae351fb986228d 2024-11-18T23:17:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971908593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971908594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971908595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T23:17:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/a5448f94a8a04399b64bfb6ec20fbb2d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/a5448f94a8a04399b64bfb6ec20fbb2d 2024-11-18T23:17:28,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/a5448f94a8a04399b64bfb6ec20fbb2d, entries=200, sequenceid=298, filesize=14.4 K 2024-11-18T23:17:28,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/bb5fe63c791c4e69a61e3907fc7770a4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bb5fe63c791c4e69a61e3907fc7770a4 2024-11-18T23:17:28,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971908620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971908620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bb5fe63c791c4e69a61e3907fc7770a4, entries=150, sequenceid=298, filesize=12.0 K 2024-11-18T23:17:28,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/220c9d1fb9c242fdb5ae351fb986228d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/220c9d1fb9c242fdb5ae351fb986228d 2024-11-18T23:17:28,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/220c9d1fb9c242fdb5ae351fb986228d, entries=150, sequenceid=298, filesize=12.0 K 2024-11-18T23:17:28,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 37a0e116df39ce62ed45cb517bbc0fb6 in 263ms, sequenceid=298, compaction requested=true 2024-11-18T23:17:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:28,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-18T23:17:28,679 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:28,680 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:28,684 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:28,684 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/A is initiating minor compaction (all files) 2024-11-18T23:17:28,684 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/A in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,685 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/46ddc2f4cccf4e4887815ab3ee470fe5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/732b54ce23784be19119d362200c1a33, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/a5448f94a8a04399b64bfb6ec20fbb2d] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=38.8 K 2024-11-18T23:17:28,685 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:28,685 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/C is initiating minor compaction (all files) 2024-11-18T23:17:28,685 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/C in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,685 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e259a97e25f1495cb624c321a715747b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/af71db8476344f56bb4298751a38abce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/220c9d1fb9c242fdb5ae351fb986228d] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=36.4 K 2024-11-18T23:17:28,685 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 46ddc2f4cccf4e4887815ab3ee470fe5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971846286 2024-11-18T23:17:28,685 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e259a97e25f1495cb624c321a715747b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971846286 2024-11-18T23:17:28,686 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 732b54ce23784be19119d362200c1a33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731971846456 2024-11-18T23:17:28,686 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting af71db8476344f56bb4298751a38abce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731971846456 2024-11-18T23:17:28,686 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a5448f94a8a04399b64bfb6ec20fbb2d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731971847195 2024-11-18T23:17:28,686 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 220c9d1fb9c242fdb5ae351fb986228d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731971847195 2024-11-18T23:17:28,702 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#A#compaction#413 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:28,703 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/d465a4fe9f224da4844d73946b8d635a is 50, key is test_row_0/A:col10/1731971847196/Put/seqid=0 2024-11-18T23:17:28,715 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:28,715 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/bf8f26b3b0d140c4bc2f93e80cedb8a7 is 50, key is test_row_0/C:col10/1731971847196/Put/seqid=0 2024-11-18T23:17:28,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742309_1485 (size=12915) 2024-11-18T23:17:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:28,754 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-18T23:17:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:28,756 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:28,763 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/d465a4fe9f224da4844d73946b8d635a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/d465a4fe9f224da4844d73946b8d635a 2024-11-18T23:17:28,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742310_1486 (size=12915) 2024-11-18T23:17:28,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/e09098144050436f86e2d62a2cd518ea is 50, key is test_row_0/A:col10/1731971848752/Put/seqid=0 2024-11-18T23:17:28,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T23:17:28,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:28,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,782 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/A of 37a0e116df39ce62ed45cb517bbc0fb6 into d465a4fe9f224da4844d73946b8d635a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:28,782 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:28,782 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/A, priority=13, startTime=1731971848679; duration=0sec 2024-11-18T23:17:28,782 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:28,782 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:A 2024-11-18T23:17:28,783 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T23:17:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,788 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:28,788 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/B is initiating minor compaction (all files) 2024-11-18T23:17:28,789 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/B in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,789 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/5ac47073e7f440d18908fc3bd1990fe2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/f78199bff8164465be3d6cb4ef630c78, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bb5fe63c791c4e69a61e3907fc7770a4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=36.4 K 2024-11-18T23:17:28,789 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ac47073e7f440d18908fc3bd1990fe2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971846286 2024-11-18T23:17:28,790 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f78199bff8164465be3d6cb4ef630c78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731971846456 2024-11-18T23:17:28,790 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5fe63c791c4e69a61e3907fc7770a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731971847195 2024-11-18T23:17:28,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742311_1487 (size=17181) 2024-11-18T23:17:28,815 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#B#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:28,815 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e4544d74f4b645ee8e17b1088b34419e is 50, key is test_row_0/B:col10/1731971847196/Put/seqid=0 2024-11-18T23:17:28,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971908796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971908797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742312_1488 (size=12915) 2024-11-18T23:17:28,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971908830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971908828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,877 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/e4544d74f4b645ee8e17b1088b34419e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e4544d74f4b645ee8e17b1088b34419e 2024-11-18T23:17:28,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971908863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,908 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/B of 37a0e116df39ce62ed45cb517bbc0fb6 into e4544d74f4b645ee8e17b1088b34419e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:28,908 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:28,908 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/B, priority=13, startTime=1731971848679; duration=0sec 2024-11-18T23:17:28,909 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:28,909 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:B 2024-11-18T23:17:28,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T23:17:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:28,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:28,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971908945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971908946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971908968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971908970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:28,997 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T23:17:29,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T23:17:29,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:29,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:29,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:29,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:29,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971909169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971909172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/e09098144050436f86e2d62a2cd518ea 2024-11-18T23:17:29,217 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/bf8f26b3b0d140c4bc2f93e80cedb8a7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bf8f26b3b0d140c4bc2f93e80cedb8a7 2024-11-18T23:17:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971909203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971909208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971909209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,236 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,244 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/C of 37a0e116df39ce62ed45cb517bbc0fb6 into bf8f26b3b0d140c4bc2f93e80cedb8a7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:29,244 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:29,244 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/C, priority=13, startTime=1731971848679; duration=0sec 2024-11-18T23:17:29,245 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:29,245 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:C 2024-11-18T23:17:29,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/02da9c2ccaf64362814836d8d2faf557 is 50, key is test_row_0/B:col10/1731971848752/Put/seqid=0 2024-11-18T23:17:29,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T23:17:29,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:29,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:29,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:29,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:29,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:29,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742313_1489 (size=12301) 2024-11-18T23:17:29,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/02da9c2ccaf64362814836d8d2faf557 2024-11-18T23:17:29,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/dedaba801f984a82b9add745b01b5d22 is 50, key is test_row_0/C:col10/1731971848752/Put/seqid=0 2024-11-18T23:17:29,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742314_1490 (size=12301) 2024-11-18T23:17:29,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/dedaba801f984a82b9add745b01b5d22 2024-11-18T23:17:29,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/e09098144050436f86e2d62a2cd518ea as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e09098144050436f86e2d62a2cd518ea 2024-11-18T23:17:29,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e09098144050436f86e2d62a2cd518ea, entries=250, sequenceid=334, filesize=16.8 K 2024-11-18T23:17:29,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/02da9c2ccaf64362814836d8d2faf557 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/02da9c2ccaf64362814836d8d2faf557 2024-11-18T23:17:29,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/02da9c2ccaf64362814836d8d2faf557, entries=150, sequenceid=334, filesize=12.0 K 2024-11-18T23:17:29,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/dedaba801f984a82b9add745b01b5d22 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/dedaba801f984a82b9add745b01b5d22 2024-11-18T23:17:29,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/dedaba801f984a82b9add745b01b5d22, entries=150, sequenceid=334, filesize=12.0 K 2024-11-18T23:17:29,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 37a0e116df39ce62ed45cb517bbc0fb6 in 668ms, sequenceid=334, compaction requested=false 2024-11-18T23:17:29,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:29,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-18T23:17:29,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:29,444 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:17:29,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:29,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:29,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fe6c79cae7b94849982c2da54ba2248f is 50, key is test_row_0/A:col10/1731971848826/Put/seqid=0 2024-11-18T23:17:29,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742315_1491 (size=12301) 2024-11-18T23:17:29,505 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fe6c79cae7b94849982c2da54ba2248f 2024-11-18T23:17:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:29,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. as already flushing 2024-11-18T23:17:29,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a46f2b86e0c446f19905b65efc5ca559 is 50, key is test_row_0/B:col10/1731971848826/Put/seqid=0 2024-11-18T23:17:29,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742316_1492 (size=12301) 2024-11-18T23:17:29,570 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a46f2b86e0c446f19905b65efc5ca559 2024-11-18T23:17:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T23:17:29,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/992863c0f68a43c9bc62ea9718c7a8b8 is 50, key is test_row_0/C:col10/1731971848826/Put/seqid=0 2024-11-18T23:17:29,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971849593, max timestamp=1731971849624Num uniques 859; 2024-11-18T23:17:29,629 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:17:29,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971849593, max timestamp=1731971849624Num uniques 859; 2024-11-18T23:17:29,631 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:17:29,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731971849593, max timestamp=1731971849624Num uniques 859; 2024-11-18T23:17:29,635 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-18T23:17:29,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742317_1493 (size=12301) 2024-11-18T23:17:29,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971909674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971909691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971909692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971909697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971909748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971909798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971909827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971909827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971909827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:29,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971910020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,048 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/992863c0f68a43c9bc62ea9718c7a8b8 2024-11-18T23:17:30,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971910037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971910038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971910038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/fe6c79cae7b94849982c2da54ba2248f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fe6c79cae7b94849982c2da54ba2248f 2024-11-18T23:17:30,075 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fe6c79cae7b94849982c2da54ba2248f, entries=150, sequenceid=347, filesize=12.0 K 2024-11-18T23:17:30,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/a46f2b86e0c446f19905b65efc5ca559 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a46f2b86e0c446f19905b65efc5ca559 2024-11-18T23:17:30,079 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a46f2b86e0c446f19905b65efc5ca559, entries=150, sequenceid=347, filesize=12.0 K 2024-11-18T23:17:30,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/992863c0f68a43c9bc62ea9718c7a8b8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/992863c0f68a43c9bc62ea9718c7a8b8 2024-11-18T23:17:30,084 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/992863c0f68a43c9bc62ea9718c7a8b8, entries=150, sequenceid=347, filesize=12.0 K 2024-11-18T23:17:30,085 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 37a0e116df39ce62ed45cb517bbc0fb6 in 641ms, sequenceid=347, compaction requested=true 2024-11-18T23:17:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:30,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-18T23:17:30,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-18T23:17:30,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-18T23:17:30,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6240 sec 2024-11-18T23:17:30,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6360 sec 2024-11-18T23:17:30,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:30,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-18T23:17:30,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:30,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:30,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:30,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:30,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:30,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:30,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/72bf90b5c9434de6a76bc933f4893827 is 50, key is test_row_0/A:col10/1731971849689/Put/seqid=0 2024-11-18T23:17:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742318_1494 (size=14741) 2024-11-18T23:17:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971910383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/72bf90b5c9434de6a76bc933f4893827 2024-11-18T23:17:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971910384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971910384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971910399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/82ad00f0773a4acdb06a43fde700a487 is 50, key is test_row_0/B:col10/1731971849689/Put/seqid=0 2024-11-18T23:17:30,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742319_1495 (size=12301) 2024-11-18T23:17:30,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/82ad00f0773a4acdb06a43fde700a487 2024-11-18T23:17:30,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/c141e3cbdd344edb8b99c59c6221c3fb is 50, key is test_row_0/C:col10/1731971849689/Put/seqid=0 2024-11-18T23:17:30,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742320_1496 (size=12301) 2024-11-18T23:17:30,488 INFO [Thread-1920 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1920. 2024-11-18T23:17:30,488 DEBUG [Thread-1920 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:30,488 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,491 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,491 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:30,498 INFO [Thread-1918 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1918. 2024-11-18T23:17:30,499 DEBUG [Thread-1918 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:30,499 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,499 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:30,500 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-11-18T23:17:30,501 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:30,501 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,501 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:30,501 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-11-18T23:17:30,501 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:30,501 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,501 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971910507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971910507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971910507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,514 INFO [Thread-1922 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1922. 2024-11-18T23:17:30,514 DEBUG [Thread-1922 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:30,514 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,514 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:30,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:30,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971910518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-18T23:17:30,608 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36740 deadline: 1731971910717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36706 deadline: 1731971910717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36732 deadline: 1731971910717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36744 deadline: 1731971910727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36780 deadline: 1731971910767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:30,767 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in 37a0e116df39ce62ed45cb517bbc0fb6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2279 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 , the old value is region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., hostname=9d96596e55a9,32899,1731971713394, seqNum=2 because the exception is null or not the one we care about 2024-11-18T23:17:30,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/c141e3cbdd344edb8b99c59c6221c3fb 2024-11-18T23:17:30,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/72bf90b5c9434de6a76bc933f4893827 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/72bf90b5c9434de6a76bc933f4893827 2024-11-18T23:17:30,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/72bf90b5c9434de6a76bc933f4893827, entries=200, sequenceid=383, filesize=14.4 K 2024-11-18T23:17:30,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/82ad00f0773a4acdb06a43fde700a487 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/82ad00f0773a4acdb06a43fde700a487 2024-11-18T23:17:30,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/82ad00f0773a4acdb06a43fde700a487, entries=150, sequenceid=383, filesize=12.0 K 2024-11-18T23:17:30,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/c141e3cbdd344edb8b99c59c6221c3fb as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/c141e3cbdd344edb8b99c59c6221c3fb 2024-11-18T23:17:30,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/c141e3cbdd344edb8b99c59c6221c3fb, entries=150, sequenceid=383, filesize=12.0 K 2024-11-18T23:17:30,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 37a0e116df39ce62ed45cb517bbc0fb6 in 559ms, sequenceid=383, compaction requested=true 2024-11-18T23:17:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:30,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:30,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:30,908 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:30,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:30,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37a0e116df39ce62ed45cb517bbc0fb6:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:30,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:30,908 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:30,909 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57138 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:30,909 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/A is initiating minor compaction (all files) 2024-11-18T23:17:30,909 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/A in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:30,909 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/d465a4fe9f224da4844d73946b8d635a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e09098144050436f86e2d62a2cd518ea, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fe6c79cae7b94849982c2da54ba2248f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/72bf90b5c9434de6a76bc933f4893827] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=55.8 K 2024-11-18T23:17:30,909 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:30,910 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/B is initiating minor compaction (all files) 2024-11-18T23:17:30,910 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/B in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:30,910 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e4544d74f4b645ee8e17b1088b34419e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/02da9c2ccaf64362814836d8d2faf557, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a46f2b86e0c446f19905b65efc5ca559, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/82ad00f0773a4acdb06a43fde700a487] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=48.7 K 2024-11-18T23:17:30,910 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d465a4fe9f224da4844d73946b8d635a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731971847195 2024-11-18T23:17:30,910 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e4544d74f4b645ee8e17b1088b34419e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731971847195 2024-11-18T23:17:30,910 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09098144050436f86e2d62a2cd518ea, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731971848569 2024-11-18T23:17:30,910 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 02da9c2ccaf64362814836d8d2faf557, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731971848591 2024-11-18T23:17:30,911 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a46f2b86e0c446f19905b65efc5ca559, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731971848792 2024-11-18T23:17:30,911 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe6c79cae7b94849982c2da54ba2248f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731971848792 2024-11-18T23:17:30,911 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ad00f0773a4acdb06a43fde700a487, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731971849663 2024-11-18T23:17:30,911 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72bf90b5c9434de6a76bc933f4893827, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731971849663 2024-11-18T23:17:30,931 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#B#compaction#425 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:30,931 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#A#compaction#426 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:30,931 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/8e8f1c6c34c9446c808a2cc9e6d31b8a is 50, key is test_row_0/B:col10/1731971849689/Put/seqid=0 2024-11-18T23:17:30,931 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/0e36bb8015d242f6bd732148b2b91332 is 50, key is test_row_0/A:col10/1731971849689/Put/seqid=0 2024-11-18T23:17:30,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742322_1498 (size=13051) 2024-11-18T23:17:30,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742321_1497 (size=13051) 2024-11-18T23:17:30,961 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/0e36bb8015d242f6bd732148b2b91332 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/0e36bb8015d242f6bd732148b2b91332 2024-11-18T23:17:30,968 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/A of 37a0e116df39ce62ed45cb517bbc0fb6 into 0e36bb8015d242f6bd732148b2b91332(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:30,969 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:30,969 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/A, priority=12, startTime=1731971850907; duration=0sec 2024-11-18T23:17:30,969 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:30,969 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:A 2024-11-18T23:17:30,969 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:30,971 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:30,971 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): 37a0e116df39ce62ed45cb517bbc0fb6/C is initiating minor compaction (all files) 2024-11-18T23:17:30,971 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37a0e116df39ce62ed45cb517bbc0fb6/C in TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:30,971 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bf8f26b3b0d140c4bc2f93e80cedb8a7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/dedaba801f984a82b9add745b01b5d22, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/992863c0f68a43c9bc62ea9718c7a8b8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/c141e3cbdd344edb8b99c59c6221c3fb] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp, totalSize=48.7 K 2024-11-18T23:17:30,973 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf8f26b3b0d140c4bc2f93e80cedb8a7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731971847195 2024-11-18T23:17:30,973 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting dedaba801f984a82b9add745b01b5d22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731971848591 2024-11-18T23:17:30,973 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 992863c0f68a43c9bc62ea9718c7a8b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731971848792 2024-11-18T23:17:30,973 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting c141e3cbdd344edb8b99c59c6221c3fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731971849663 2024-11-18T23:17:31,009 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37a0e116df39ce62ed45cb517bbc0fb6#C#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:31,009 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/57b68745b09a49679cc32320bc447580 is 50, key is test_row_0/C:col10/1731971849689/Put/seqid=0 2024-11-18T23:17:31,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742323_1499 (size=13051) 2024-11-18T23:17:31,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:17:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:31,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:31,038 INFO [Thread-1911 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1911. 2024-11-18T23:17:31,038 DEBUG [Thread-1911 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:31,038 DEBUG [Thread-1911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,038 DEBUG [Thread-1911 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,039 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:31,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:31,045 INFO [Thread-1909 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1909. 2024-11-18T23:17:31,045 DEBUG [Thread-1909 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:31,045 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,045 DEBUG [Thread-1909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:31,048 INFO [Thread-1907 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1907. 2024-11-18T23:17:31,048 DEBUG [Thread-1907 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:31,048 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,048 DEBUG [Thread-1907 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:31,049 INFO [Thread-1903 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1903. 2024-11-18T23:17:31,049 DEBUG [Thread-1903 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:31,049 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,049 DEBUG [Thread-1903 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:31,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:31,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/5990cd5630174759a5680619011ef3bb is 50, key is test_row_0/A:col10/1731971851030/Put/seqid=0 2024-11-18T23:17:31,080 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/57b68745b09a49679cc32320bc447580 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/57b68745b09a49679cc32320bc447580 2024-11-18T23:17:31,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742324_1500 (size=9857) 2024-11-18T23:17:31,098 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/C of 37a0e116df39ce62ed45cb517bbc0fb6 into 57b68745b09a49679cc32320bc447580(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:31,098 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:31,098 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/C, priority=12, startTime=1731971850908; duration=0sec 2024-11-18T23:17:31,098 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:31,098 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:C 2024-11-18T23:17:31,373 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/8e8f1c6c34c9446c808a2cc9e6d31b8a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/8e8f1c6c34c9446c808a2cc9e6d31b8a 2024-11-18T23:17:31,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37a0e116df39ce62ed45cb517bbc0fb6/B of 37a0e116df39ce62ed45cb517bbc0fb6 into 8e8f1c6c34c9446c808a2cc9e6d31b8a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:31,385 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:31,385 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., storeName=37a0e116df39ce62ed45cb517bbc0fb6/B, priority=12, startTime=1731971850908; duration=0sec 2024-11-18T23:17:31,385 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:31,385 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37a0e116df39ce62ed45cb517bbc0fb6:B 2024-11-18T23:17:31,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/5990cd5630174759a5680619011ef3bb 2024-11-18T23:17:31,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/ffb2151cd9194e52903a5bb94b9522f2 is 50, key is test_row_0/B:col10/1731971851030/Put/seqid=0 2024-11-18T23:17:31,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742325_1501 (size=9857) 2024-11-18T23:17:31,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/ffb2151cd9194e52903a5bb94b9522f2 2024-11-18T23:17:31,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/f5ff989b611e47b499fccd2afcd055cd is 50, key is test_row_0/C:col10/1731971851030/Put/seqid=0 2024-11-18T23:17:32,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742326_1502 (size=9857) 2024-11-18T23:17:32,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/f5ff989b611e47b499fccd2afcd055cd 2024-11-18T23:17:32,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/5990cd5630174759a5680619011ef3bb as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/5990cd5630174759a5680619011ef3bb 2024-11-18T23:17:32,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/5990cd5630174759a5680619011ef3bb, entries=100, sequenceid=395, filesize=9.6 K 2024-11-18T23:17:32,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/ffb2151cd9194e52903a5bb94b9522f2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ffb2151cd9194e52903a5bb94b9522f2 2024-11-18T23:17:32,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ffb2151cd9194e52903a5bb94b9522f2, entries=100, sequenceid=395, filesize=9.6 K 2024-11-18T23:17:32,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/f5ff989b611e47b499fccd2afcd055cd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/f5ff989b611e47b499fccd2afcd055cd 2024-11-18T23:17:32,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/f5ff989b611e47b499fccd2afcd055cd, entries=100, sequenceid=395, filesize=9.6 K 2024-11-18T23:17:32,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 37a0e116df39ce62ed45cb517bbc0fb6 in 1037ms, sequenceid=395, compaction requested=false 2024-11-18T23:17:32,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37a0e116df39ce62ed45cb517bbc0fb6: 2024-11-18T23:17:32,796 INFO [Thread-1905 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1905. 2024-11-18T23:17:32,796 DEBUG [Thread-1905 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:32,796 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:32,796 DEBUG [Thread-1905 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:32,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 649 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1942 rows 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 635 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1905 rows 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 648 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1939 rows 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 644 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1930 rows 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 639 2024-11-18T23:17:32,797 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1917 rows 2024-11-18T23:17:32,797 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:17:32,797 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:17:32,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:32,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:32,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:32,798 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T23:17:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:32,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T23:17:32,812 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971852812"}]},"ts":"1731971852812"} 2024-11-18T23:17:32,824 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T23:17:32,824 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T23:17:32,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:17:32,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, UNASSIGN}] 2024-11-18T23:17:32,849 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, UNASSIGN 2024-11-18T23:17:32,850 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=37a0e116df39ce62ed45cb517bbc0fb6, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:32,853 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=127, ppid=126, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, UNASSIGN because future has completed 2024-11-18T23:17:32,854 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:17:32,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:17:32,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T23:17:33,009 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(122): Close 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:33,009 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:17:33,009 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1722): Closing 37a0e116df39ce62ed45cb517bbc0fb6, disabling compactions & flushes 2024-11-18T23:17:33,009 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:33,009 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:33,009 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. after waiting 0 ms 2024-11-18T23:17:33,009 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:33,009 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(2902): Flushing 37a0e116df39ce62ed45cb517bbc0fb6 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-18T23:17:33,010 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=A 2024-11-18T23:17:33,010 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:33,010 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=B 2024-11-18T23:17:33,010 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:33,010 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 37a0e116df39ce62ed45cb517bbc0fb6, store=C 2024-11-18T23:17:33,010 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:33,018 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/4a88e88f9a2f4c38bf590a56c5e06100 is 50, key is test_row_0/A:col10/1731971851039/Put/seqid=0 2024-11-18T23:17:33,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742327_1503 (size=12301) 2024-11-18T23:17:33,033 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/4a88e88f9a2f4c38bf590a56c5e06100 2024-11-18T23:17:33,052 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7ba77adbad0143b88f5c922d4e74eee9 is 50, key is test_row_0/B:col10/1731971851039/Put/seqid=0 2024-11-18T23:17:33,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742328_1504 (size=12301) 2024-11-18T23:17:33,082 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7ba77adbad0143b88f5c922d4e74eee9 2024-11-18T23:17:33,093 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/b8e7294948fc4bb39283672fe4aa91ec is 50, key is test_row_0/C:col10/1731971851039/Put/seqid=0 2024-11-18T23:17:33,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742329_1505 (size=12301) 2024-11-18T23:17:33,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T23:17:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T23:17:33,503 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/b8e7294948fc4bb39283672fe4aa91ec 2024-11-18T23:17:33,508 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/A/4a88e88f9a2f4c38bf590a56c5e06100 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a88e88f9a2f4c38bf590a56c5e06100 2024-11-18T23:17:33,518 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a88e88f9a2f4c38bf590a56c5e06100, entries=150, sequenceid=403, filesize=12.0 K 2024-11-18T23:17:33,519 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/B/7ba77adbad0143b88f5c922d4e74eee9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7ba77adbad0143b88f5c922d4e74eee9 2024-11-18T23:17:33,523 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7ba77adbad0143b88f5c922d4e74eee9, entries=150, sequenceid=403, filesize=12.0 K 2024-11-18T23:17:33,524 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/.tmp/C/b8e7294948fc4bb39283672fe4aa91ec as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/b8e7294948fc4bb39283672fe4aa91ec 2024-11-18T23:17:33,533 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/b8e7294948fc4bb39283672fe4aa91ec, entries=150, sequenceid=403, filesize=12.0 K 2024-11-18T23:17:33,533 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 37a0e116df39ce62ed45cb517bbc0fb6 in 524ms, sequenceid=403, compaction requested=true 2024-11-18T23:17:33,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ca0801d504cc4639be7a957eb3765516, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fbaac120514e44cc80346b023b29c4f1, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/6c71ce65064b41f7ae841e78669c6059, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/70000a3059464dd8bb11bc8f10730b9e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/cb829408f7b9412b8ee6985fe1d9c08b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/185740bc6d6243bbb4c912ecffb2f33e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a0ca9bde52c4546a2f1c5fa44e9d00f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fdf4da3064494ac59f9f9c860f7e1c74, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/3d21124378f645758d4cecc6cc032ec3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fb8b83407b2a42c79766d72db4f0d675, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/24018519cde14f5a9b2fd53702063168, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1a1e5719030e4d21ab2159bf7c1986da, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/83d5cc1787fb40f193ec1bd26e7e8248, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/912d2a109dda4f239be064edbf0ec736, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1d02479e090d44dea029df3618657772, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/46ddc2f4cccf4e4887815ab3ee470fe5, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/732b54ce23784be19119d362200c1a33, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/a5448f94a8a04399b64bfb6ec20fbb2d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/d465a4fe9f224da4844d73946b8d635a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e09098144050436f86e2d62a2cd518ea, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fe6c79cae7b94849982c2da54ba2248f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/72bf90b5c9434de6a76bc933f4893827] to archive 2024-11-18T23:17:33,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:33,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ca0801d504cc4639be7a957eb3765516 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/ca0801d504cc4639be7a957eb3765516 2024-11-18T23:17:33,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fbaac120514e44cc80346b023b29c4f1 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fbaac120514e44cc80346b023b29c4f1 2024-11-18T23:17:33,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/6c71ce65064b41f7ae841e78669c6059 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/6c71ce65064b41f7ae841e78669c6059 2024-11-18T23:17:33,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/70000a3059464dd8bb11bc8f10730b9e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/70000a3059464dd8bb11bc8f10730b9e 2024-11-18T23:17:33,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/cb829408f7b9412b8ee6985fe1d9c08b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/cb829408f7b9412b8ee6985fe1d9c08b 2024-11-18T23:17:33,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/185740bc6d6243bbb4c912ecffb2f33e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/185740bc6d6243bbb4c912ecffb2f33e 2024-11-18T23:17:33,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a0ca9bde52c4546a2f1c5fa44e9d00f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a0ca9bde52c4546a2f1c5fa44e9d00f 2024-11-18T23:17:33,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fdf4da3064494ac59f9f9c860f7e1c74 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fdf4da3064494ac59f9f9c860f7e1c74 2024-11-18T23:17:33,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/3d21124378f645758d4cecc6cc032ec3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/3d21124378f645758d4cecc6cc032ec3 2024-11-18T23:17:33,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fb8b83407b2a42c79766d72db4f0d675 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fb8b83407b2a42c79766d72db4f0d675 2024-11-18T23:17:33,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/24018519cde14f5a9b2fd53702063168 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/24018519cde14f5a9b2fd53702063168 2024-11-18T23:17:33,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1a1e5719030e4d21ab2159bf7c1986da to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1a1e5719030e4d21ab2159bf7c1986da 2024-11-18T23:17:33,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/83d5cc1787fb40f193ec1bd26e7e8248 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/83d5cc1787fb40f193ec1bd26e7e8248 2024-11-18T23:17:33,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/912d2a109dda4f239be064edbf0ec736 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/912d2a109dda4f239be064edbf0ec736 2024-11-18T23:17:33,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1d02479e090d44dea029df3618657772 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/1d02479e090d44dea029df3618657772 2024-11-18T23:17:33,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/46ddc2f4cccf4e4887815ab3ee470fe5 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/46ddc2f4cccf4e4887815ab3ee470fe5 2024-11-18T23:17:33,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/732b54ce23784be19119d362200c1a33 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/732b54ce23784be19119d362200c1a33 2024-11-18T23:17:33,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/a5448f94a8a04399b64bfb6ec20fbb2d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/a5448f94a8a04399b64bfb6ec20fbb2d 2024-11-18T23:17:33,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/d465a4fe9f224da4844d73946b8d635a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/d465a4fe9f224da4844d73946b8d635a 2024-11-18T23:17:33,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e09098144050436f86e2d62a2cd518ea to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/e09098144050436f86e2d62a2cd518ea 2024-11-18T23:17:33,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fe6c79cae7b94849982c2da54ba2248f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/fe6c79cae7b94849982c2da54ba2248f 2024-11-18T23:17:33,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/72bf90b5c9434de6a76bc933f4893827 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/72bf90b5c9434de6a76bc933f4893827 2024-11-18T23:17:33,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ee09733345064e97a3299e9e1000ee45, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/c15ce5d2f901471090f6023fdeae4bcf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/92dd4d6e288d4bc7af0d11c288310701, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e829a956af6248fdb85f44145cfd2794, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/3efa518187bf4be6ad823b533e7987f6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/749d1a059f514075bd7eac43b1ecd901, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/0e20b014821b4d61b228baf363ff2e16, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/4f8c68f58534418fb289a1eab7b859b3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7225a7b7284f4a90b927995545eb1328, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a252021a061e4237ac66c77177d9832e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bbb8612a88314cf08d824da186286a3a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e89d107e499f4538a437fb74c87323f4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a3259e8eda9941f2858e6056cd0062f4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7add1a88155d438b829651dcfe21b158, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/5ac47073e7f440d18908fc3bd1990fe2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/1574b8b319864fc89dde5627ba3c13a6, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/f78199bff8164465be3d6cb4ef630c78, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e4544d74f4b645ee8e17b1088b34419e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bb5fe63c791c4e69a61e3907fc7770a4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/02da9c2ccaf64362814836d8d2faf557, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a46f2b86e0c446f19905b65efc5ca559, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/82ad00f0773a4acdb06a43fde700a487] to archive 2024-11-18T23:17:33,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:33,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ee09733345064e97a3299e9e1000ee45 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ee09733345064e97a3299e9e1000ee45 2024-11-18T23:17:33,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/c15ce5d2f901471090f6023fdeae4bcf to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/c15ce5d2f901471090f6023fdeae4bcf 2024-11-18T23:17:33,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/92dd4d6e288d4bc7af0d11c288310701 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/92dd4d6e288d4bc7af0d11c288310701 2024-11-18T23:17:33,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e829a956af6248fdb85f44145cfd2794 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e829a956af6248fdb85f44145cfd2794 2024-11-18T23:17:33,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/3efa518187bf4be6ad823b533e7987f6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/3efa518187bf4be6ad823b533e7987f6 2024-11-18T23:17:33,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/749d1a059f514075bd7eac43b1ecd901 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/749d1a059f514075bd7eac43b1ecd901 2024-11-18T23:17:33,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/0e20b014821b4d61b228baf363ff2e16 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/0e20b014821b4d61b228baf363ff2e16 2024-11-18T23:17:33,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/4f8c68f58534418fb289a1eab7b859b3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/4f8c68f58534418fb289a1eab7b859b3 2024-11-18T23:17:33,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7225a7b7284f4a90b927995545eb1328 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7225a7b7284f4a90b927995545eb1328 2024-11-18T23:17:33,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a252021a061e4237ac66c77177d9832e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a252021a061e4237ac66c77177d9832e 2024-11-18T23:17:33,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bbb8612a88314cf08d824da186286a3a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bbb8612a88314cf08d824da186286a3a 2024-11-18T23:17:33,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e89d107e499f4538a437fb74c87323f4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e89d107e499f4538a437fb74c87323f4 2024-11-18T23:17:33,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a3259e8eda9941f2858e6056cd0062f4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a3259e8eda9941f2858e6056cd0062f4 2024-11-18T23:17:33,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7add1a88155d438b829651dcfe21b158 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7add1a88155d438b829651dcfe21b158 2024-11-18T23:17:33,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/5ac47073e7f440d18908fc3bd1990fe2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/5ac47073e7f440d18908fc3bd1990fe2 2024-11-18T23:17:33,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/1574b8b319864fc89dde5627ba3c13a6 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/1574b8b319864fc89dde5627ba3c13a6 2024-11-18T23:17:33,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/f78199bff8164465be3d6cb4ef630c78 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/f78199bff8164465be3d6cb4ef630c78 2024-11-18T23:17:33,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e4544d74f4b645ee8e17b1088b34419e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/e4544d74f4b645ee8e17b1088b34419e 2024-11-18T23:17:33,627 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bb5fe63c791c4e69a61e3907fc7770a4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/bb5fe63c791c4e69a61e3907fc7770a4 2024-11-18T23:17:33,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/02da9c2ccaf64362814836d8d2faf557 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/02da9c2ccaf64362814836d8d2faf557 2024-11-18T23:17:33,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a46f2b86e0c446f19905b65efc5ca559 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/a46f2b86e0c446f19905b65efc5ca559 2024-11-18T23:17:33,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/82ad00f0773a4acdb06a43fde700a487 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/82ad00f0773a4acdb06a43fde700a487 2024-11-18T23:17:33,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bb1feb17120f472d8d96ed36d435c50c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e8cae4cc22e445ca93c11d7aa23d9824, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/03284bb54b8a4b98aec302eb48367eeb, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/24d4f5f1b5144f8d8b553e36c7bf92d8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/cfd8c26570014e888f0b702f9a1b5863, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2f57b1faf76a4611b394ef735d8bdf24, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6068b0df05bc414398e00a86034d7b3e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/65217e9f662d445d998c9ba308c9a53c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d0417003a9134e3f82f7e7a70b6422df, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/08aedd1533a34091917d00ff2ce3f8fc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6e79fe0eeac848d3b0c29f551704b601, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/4e0c406fe1574cd687737f1372d71b43, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/5cd8ac1051fc46bc9f4dfe50d10c5b77, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/acf323ee0a8447f5871bcf762f17a037, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e259a97e25f1495cb624c321a715747b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d7888d6119d14536a72f6f7885c11e50, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/af71db8476344f56bb4298751a38abce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bf8f26b3b0d140c4bc2f93e80cedb8a7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/220c9d1fb9c242fdb5ae351fb986228d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/dedaba801f984a82b9add745b01b5d22, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/992863c0f68a43c9bc62ea9718c7a8b8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/c141e3cbdd344edb8b99c59c6221c3fb] to archive 2024-11-18T23:17:33,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:17:33,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bb1feb17120f472d8d96ed36d435c50c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bb1feb17120f472d8d96ed36d435c50c 2024-11-18T23:17:33,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e8cae4cc22e445ca93c11d7aa23d9824 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e8cae4cc22e445ca93c11d7aa23d9824 2024-11-18T23:17:33,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/03284bb54b8a4b98aec302eb48367eeb to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/03284bb54b8a4b98aec302eb48367eeb 2024-11-18T23:17:33,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/24d4f5f1b5144f8d8b553e36c7bf92d8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/24d4f5f1b5144f8d8b553e36c7bf92d8 2024-11-18T23:17:33,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/cfd8c26570014e888f0b702f9a1b5863 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/cfd8c26570014e888f0b702f9a1b5863 2024-11-18T23:17:33,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2f57b1faf76a4611b394ef735d8bdf24 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/2f57b1faf76a4611b394ef735d8bdf24 2024-11-18T23:17:33,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6068b0df05bc414398e00a86034d7b3e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6068b0df05bc414398e00a86034d7b3e 2024-11-18T23:17:33,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/65217e9f662d445d998c9ba308c9a53c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/65217e9f662d445d998c9ba308c9a53c 2024-11-18T23:17:33,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d0417003a9134e3f82f7e7a70b6422df to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d0417003a9134e3f82f7e7a70b6422df 2024-11-18T23:17:33,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/08aedd1533a34091917d00ff2ce3f8fc to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/08aedd1533a34091917d00ff2ce3f8fc 2024-11-18T23:17:33,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6e79fe0eeac848d3b0c29f551704b601 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/6e79fe0eeac848d3b0c29f551704b601 2024-11-18T23:17:33,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/4e0c406fe1574cd687737f1372d71b43 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/4e0c406fe1574cd687737f1372d71b43 2024-11-18T23:17:33,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/5cd8ac1051fc46bc9f4dfe50d10c5b77 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/5cd8ac1051fc46bc9f4dfe50d10c5b77 2024-11-18T23:17:33,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/acf323ee0a8447f5871bcf762f17a037 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/acf323ee0a8447f5871bcf762f17a037 2024-11-18T23:17:33,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e259a97e25f1495cb624c321a715747b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/e259a97e25f1495cb624c321a715747b 2024-11-18T23:17:33,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d7888d6119d14536a72f6f7885c11e50 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/d7888d6119d14536a72f6f7885c11e50 2024-11-18T23:17:33,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/af71db8476344f56bb4298751a38abce to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/af71db8476344f56bb4298751a38abce 2024-11-18T23:17:33,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bf8f26b3b0d140c4bc2f93e80cedb8a7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/bf8f26b3b0d140c4bc2f93e80cedb8a7 2024-11-18T23:17:33,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/220c9d1fb9c242fdb5ae351fb986228d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/220c9d1fb9c242fdb5ae351fb986228d 2024-11-18T23:17:33,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/dedaba801f984a82b9add745b01b5d22 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/dedaba801f984a82b9add745b01b5d22 2024-11-18T23:17:33,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/992863c0f68a43c9bc62ea9718c7a8b8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/992863c0f68a43c9bc62ea9718c7a8b8 2024-11-18T23:17:33,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/c141e3cbdd344edb8b99c59c6221c3fb to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/c141e3cbdd344edb8b99c59c6221c3fb 2024-11-18T23:17:33,669 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/recovered.edits/406.seqid, newMaxSeqId=406, maxSeqId=1 2024-11-18T23:17:33,670 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. 2024-11-18T23:17:33,670 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] regionserver.HRegion(1676): Region close journal for 37a0e116df39ce62ed45cb517bbc0fb6: Waiting for close lock at 1731971853009Running coprocessor pre-close hooks at 1731971853009Disabling compacts and flushes for region at 1731971853009Disabling writes for close at 1731971853009Obtaining lock to block concurrent updates at 1731971853009Preparing flush snapshotting stores in 37a0e116df39ce62ed45cb517bbc0fb6 at 1731971853009Finished memstore snapshotting TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731971853010 (+1 ms)Flushing stores of TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6. at 1731971853012 (+2 ms)Flushing 37a0e116df39ce62ed45cb517bbc0fb6/A: creating writer at 1731971853012Flushing 37a0e116df39ce62ed45cb517bbc0fb6/A: appending metadata at 1731971853018 (+6 ms)Flushing 37a0e116df39ce62ed45cb517bbc0fb6/A: closing flushed file at 1731971853018Flushing 37a0e116df39ce62ed45cb517bbc0fb6/B: creating writer at 1731971853045 (+27 ms)Flushing 37a0e116df39ce62ed45cb517bbc0fb6/B: appending metadata at 1731971853052 (+7 ms)Flushing 37a0e116df39ce62ed45cb517bbc0fb6/B: closing flushed file at 1731971853052Flushing 37a0e116df39ce62ed45cb517bbc0fb6/C: creating writer at 1731971853087 (+35 ms)Flushing 37a0e116df39ce62ed45cb517bbc0fb6/C: appending metadata at 1731971853093 (+6 ms)Flushing 37a0e116df39ce62ed45cb517bbc0fb6/C: closing flushed file at 1731971853093Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a1be4b9: reopening flushed file at 1731971853507 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26c31a94: reopening flushed file at 1731971853518 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2da94ec0: reopening flushed file at 1731971853523 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 37a0e116df39ce62ed45cb517bbc0fb6 in 524ms, sequenceid=403, compaction requested=true at 1731971853533 (+10 ms)Writing region close event to WAL at 1731971853667 (+134 ms)Running coprocessor post-close hooks at 1731971853670 (+3 ms)Closed at 1731971853670 2024-11-18T23:17:33,672 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=128}] handler.UnassignRegionHandler(157): Closed 37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:33,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=127 updating hbase:meta row=37a0e116df39ce62ed45cb517bbc0fb6, regionState=CLOSED 2024-11-18T23:17:33,674 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=128, ppid=127, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:17:33,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-18T23:17:33,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseRegionProcedure 37a0e116df39ce62ed45cb517bbc0fb6, server=9d96596e55a9,32899,1731971713394 in 821 msec 2024-11-18T23:17:33,680 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=127, resume processing ppid=126 2024-11-18T23:17:33,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, ppid=126, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37a0e116df39ce62ed45cb517bbc0fb6, UNASSIGN in 834 msec 2024-11-18T23:17:33,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-18T23:17:33,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 853 msec 2024-11-18T23:17:33,686 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971853686"}]},"ts":"1731971853686"} 2024-11-18T23:17:33,693 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T23:17:33,693 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T23:17:33,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 898 msec 2024-11-18T23:17:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-18T23:17:33,949 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:33,950 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T23:17:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:33,955 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=129, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T23:17:33,956 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=129, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:33,990 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:34,004 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/recovered.edits] 2024-11-18T23:17:34,008 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/0e36bb8015d242f6bd732148b2b91332 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/0e36bb8015d242f6bd732148b2b91332 2024-11-18T23:17:34,009 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a88e88f9a2f4c38bf590a56c5e06100 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/4a88e88f9a2f4c38bf590a56c5e06100 2024-11-18T23:17:34,011 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/5990cd5630174759a5680619011ef3bb to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/A/5990cd5630174759a5680619011ef3bb 2024-11-18T23:17:34,026 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7ba77adbad0143b88f5c922d4e74eee9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/7ba77adbad0143b88f5c922d4e74eee9 2024-11-18T23:17:34,027 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/8e8f1c6c34c9446c808a2cc9e6d31b8a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/8e8f1c6c34c9446c808a2cc9e6d31b8a 2024-11-18T23:17:34,029 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ffb2151cd9194e52903a5bb94b9522f2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/B/ffb2151cd9194e52903a5bb94b9522f2 2024-11-18T23:17:34,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/57b68745b09a49679cc32320bc447580 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/57b68745b09a49679cc32320bc447580 2024-11-18T23:17:34,033 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/b8e7294948fc4bb39283672fe4aa91ec to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/b8e7294948fc4bb39283672fe4aa91ec 2024-11-18T23:17:34,034 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/f5ff989b611e47b499fccd2afcd055cd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/C/f5ff989b611e47b499fccd2afcd055cd 2024-11-18T23:17:34,038 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/recovered.edits/406.seqid to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6/recovered.edits/406.seqid 2024-11-18T23:17:34,039 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/37a0e116df39ce62ed45cb517bbc0fb6 2024-11-18T23:17:34,039 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T23:17:34,047 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=129, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:34,056 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T23:17:34,066 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T23:17:34,070 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=129, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:34,070 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T23:17:34,070 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731971854070"}]},"ts":"9223372036854775807"} 2024-11-18T23:17:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T23:17:34,073 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T23:17:34,073 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 37a0e116df39ce62ed45cb517bbc0fb6, NAME => 'TestAcidGuarantees,,1731971829115.37a0e116df39ce62ed45cb517bbc0fb6.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T23:17:34,073 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T23:17:34,074 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731971854073"}]},"ts":"9223372036854775807"} 2024-11-18T23:17:34,078 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T23:17:34,080 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=129, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:34,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 130 msec 2024-11-18T23:17:34,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-18T23:17:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T23:17:34,283 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:34,316 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 251), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1623 (was 1665), ProcessCount=11 (was 11), AvailableMemoryMB=4351 (was 6543) 2024-11-18T23:17:34,328 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=1623, ProcessCount=11, AvailableMemoryMB=4351 2024-11-18T23:17:34,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:17:34,330 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:17:34,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:34,333 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T23:17:34,333 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:34,334 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 130 2024-11-18T23:17:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-18T23:17:34,335 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T23:17:34,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742330_1506 (size=1061) 2024-11-18T23:17:34,367 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff 2024-11-18T23:17:34,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742331_1507 (size=53) 2024-11-18T23:17:34,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-18T23:17:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-18T23:17:34,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:17:34,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b28daae718a8c78c9acc9f3a87d1c727, disabling compactions & flushes 2024-11-18T23:17:34,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:34,804 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:34,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. after waiting 0 ms 2024-11-18T23:17:34,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:34,805 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:34,805 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b28daae718a8c78c9acc9f3a87d1c727: Waiting for close lock at 1731971854804Disabling compacts and flushes for region at 1731971854804Disabling writes for close at 1731971854805 (+1 ms)Writing region close event to WAL at 1731971854805Closed at 1731971854805 2024-11-18T23:17:34,806 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T23:17:34,807 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731971854806"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731971854806"}]},"ts":"1731971854806"} 2024-11-18T23:17:34,810 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T23:17:34,811 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T23:17:34,811 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971854811"}]},"ts":"1731971854811"} 2024-11-18T23:17:34,814 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-18T23:17:34,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, ASSIGN}] 2024-11-18T23:17:34,816 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, ASSIGN 2024-11-18T23:17:34,822 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, ASSIGN; state=OFFLINE, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=false 2024-11-18T23:17:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-18T23:17:34,975 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:34,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, ASSIGN because future has completed 2024-11-18T23:17:34,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:17:35,149 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:35,150 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7752): Opening region: {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:17:35,150 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,150 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:17:35,150 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7794): checking encryption for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,151 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(7797): checking classloading for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,154 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,165 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:35,165 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b28daae718a8c78c9acc9f3a87d1c727 columnFamilyName A 2024-11-18T23:17:35,165 DEBUG [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:35,170 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(327): Store=b28daae718a8c78c9acc9f3a87d1c727/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:35,170 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,172 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:35,173 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b28daae718a8c78c9acc9f3a87d1c727 columnFamilyName B 2024-11-18T23:17:35,173 DEBUG [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:35,174 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(327): Store=b28daae718a8c78c9acc9f3a87d1c727/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:35,174 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,177 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:35,177 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b28daae718a8c78c9acc9f3a87d1c727 columnFamilyName C 2024-11-18T23:17:35,177 DEBUG [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:35,178 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(327): Store=b28daae718a8c78c9acc9f3a87d1c727/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:35,178 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:35,178 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1038): replaying wal for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,184 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,185 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,192 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1048): stopping wal replay for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,193 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1060): Cleaning up temporary data for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,195 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:17:35,202 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1093): writing seq id for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,209 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T23:17:35,210 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1114): Opened b28daae718a8c78c9acc9f3a87d1c727; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60007486, jitterRate=-0.10581877827644348}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:17:35,210 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:35,210 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegion(1006): Region open journal for b28daae718a8c78c9acc9f3a87d1c727: Running coprocessor pre-open hook at 1731971855151Writing region info on filesystem at 1731971855151Initializing all the Stores at 1731971855152 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971855152Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971855152Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971855152Cleaning up temporary data from old regions at 1731971855193 (+41 ms)Running coprocessor post-open hooks at 1731971855210 (+17 ms)Region opened successfully at 1731971855210 2024-11-18T23:17:35,211 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., pid=132, masterSystemTime=1731971855134 2024-11-18T23:17:35,215 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:35,215 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=132}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:35,221 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=OPEN, openSeqNum=2, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:35,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; OpenRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:17:35,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-18T23:17:35,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; OpenRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 in 245 msec 2024-11-18T23:17:35,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-18T23:17:35,235 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, ASSIGN in 413 msec 2024-11-18T23:17:35,237 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T23:17:35,237 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971855237"}]},"ts":"1731971855237"} 2024-11-18T23:17:35,241 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-18T23:17:35,247 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=130, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T23:17:35,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 917 msec 2024-11-18T23:17:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-11-18T23:17:35,470 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:35,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:35,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:35,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:35,474 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5357c2c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:35,491 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe3be9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:35,493 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:35,495 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:35,496 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:35,500 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T23:17:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9d96596e55a9,42339,1731971712494 2024-11-18T23:17:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4da07a9d 2024-11-18T23:17:35,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T23:17:35,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T23:17:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-18T23:17:35,508 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T23:17:35,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:35,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742332_1508 (size=1097) 2024-11-18T23:17:36,003 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-18T23:17:36,004 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-18T23:17:36,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:17:36,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, REOPEN/MOVE}] 2024-11-18T23:17:36,028 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, REOPEN/MOVE 2024-11-18T23:17:36,037 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, REOPEN/MOVE because future has completed 2024-11-18T23:17:36,044 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:17:36,044 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:17:36,202 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,202 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:17:36,202 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing b28daae718a8c78c9acc9f3a87d1c727, disabling compactions & flushes 2024-11-18T23:17:36,202 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,202 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,202 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. after waiting 0 ms 2024-11-18T23:17:36,202 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,217 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-18T23:17:36,218 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,218 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for b28daae718a8c78c9acc9f3a87d1c727: Waiting for close lock at 1731971856202Running coprocessor pre-close hooks at 1731971856202Disabling compacts and flushes for region at 1731971856202Disabling writes for close at 1731971856202Writing region close event to WAL at 1731971856212 (+10 ms)Running coprocessor post-close hooks at 1731971856218 (+6 ms)Closed at 1731971856218 2024-11-18T23:17:36,218 WARN [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionServer(3299): Not adding moved region record: b28daae718a8c78c9acc9f3a87d1c727 to self. 2024-11-18T23:17:36,225 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,225 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=CLOSED 2024-11-18T23:17:36,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:17:36,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-18T23:17:36,233 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 in 185 msec 2024-11-18T23:17:36,234 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, REOPEN/MOVE; state=CLOSED, location=9d96596e55a9,32899,1731971713394; forceNewPlan=false, retain=true 2024-11-18T23:17:36,384 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=OPENING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,389 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, REOPEN/MOVE because future has completed 2024-11-18T23:17:36,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:17:36,543 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,543 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7752): Opening region: {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} 2024-11-18T23:17:36,543 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,543 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T23:17:36,544 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7794): checking encryption for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,544 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(7797): checking classloading for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,545 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,545 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:36,545 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b28daae718a8c78c9acc9f3a87d1c727 columnFamilyName A 2024-11-18T23:17:36,546 DEBUG [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:36,546 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(327): Store=b28daae718a8c78c9acc9f3a87d1c727/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:36,547 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,547 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:36,547 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b28daae718a8c78c9acc9f3a87d1c727 columnFamilyName B 2024-11-18T23:17:36,547 DEBUG [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:36,548 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(327): Store=b28daae718a8c78c9acc9f3a87d1c727/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:36,548 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,548 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-18T23:17:36,548 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b28daae718a8c78c9acc9f3a87d1c727 columnFamilyName C 2024-11-18T23:17:36,548 DEBUG [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:36,549 INFO [StoreOpener-b28daae718a8c78c9acc9f3a87d1c727-1 {}] regionserver.HStore(327): Store=b28daae718a8c78c9acc9f3a87d1c727/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T23:17:36,549 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,549 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1038): replaying wal for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,549 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,550 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,551 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1048): stopping wal replay for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,551 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1060): Cleaning up temporary data for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,551 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-18T23:17:36,553 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1093): writing seq id for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,553 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1114): Opened b28daae718a8c78c9acc9f3a87d1c727; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66755457, jitterRate=-0.005266174674034119}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-18T23:17:36,553 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,554 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegion(1006): Region open journal for b28daae718a8c78c9acc9f3a87d1c727: Running coprocessor pre-open hook at 1731971856544Writing region info on filesystem at 1731971856544Initializing all the Stores at 1731971856544Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971856544Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971856544Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731971856544Cleaning up temporary data from old regions at 1731971856551 (+7 ms)Running coprocessor post-open hooks at 1731971856553 (+2 ms)Region opened successfully at 1731971856554 (+1 ms) 2024-11-18T23:17:36,555 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., pid=137, masterSystemTime=1731971856541 2024-11-18T23:17:36,557 DEBUG [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,558 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=OPEN, openSeqNum=5, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,558 INFO [RS_OPEN_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_OPEN_REGION, pid=137}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,559 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:17:36,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=135 2024-11-18T23:17:36,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 in 171 msec 2024-11-18T23:17:36,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-18T23:17:36,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, REOPEN/MOVE in 538 msec 2024-11-18T23:17:36,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-18T23:17:36,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 557 msec 2024-11-18T23:17:36,568 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0580 sec 2024-11-18T23:17:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-18T23:17:36,569 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:36,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40503a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,570 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dcaac00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,572 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,573 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2bf826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f72e1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,574 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62cd538f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,576 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48641189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47173f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,578 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48568a93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b124bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20059666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5173843b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,584 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,585 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d8e862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,586 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ebebfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ff027c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,589 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7141e9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26817296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2637386d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,592 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325bfe61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d944cb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,594 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2455dde6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,595 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,596 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7417dfb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,596 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,598 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,598 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@504a04d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,599 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b93642c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,599 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,599 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,600 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c010e53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,602 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,603 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6160073b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@544c9525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,604 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9d96596e55a9,42339,-1 for getting cluster id 2024-11-18T23:17:36,604 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T23:17:36,605 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '312958eb-d8b4-4384-a210-556b66aba6db' 2024-11-18T23:17:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T23:17:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "312958eb-d8b4-4384-a210-556b66aba6db" 2024-11-18T23:17:36,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e66d97d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9d96596e55a9,42339,-1] 2024-11-18T23:17:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T23:17:36,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:36,607 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T23:17:36,608 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773fd8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T23:17:36,608 DEBUG [Thread-2292 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,611 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,616 DEBUG [Thread-2290 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,617 DEBUG [Thread-2279 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,620 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,634 DEBUG [Thread-2281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,637 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,638 DEBUG [Thread-2287 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,638 DEBUG [Thread-2294 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,639 DEBUG [Thread-2287 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,642 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,644 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:17:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:36,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:36,647 DEBUG [Thread-2296 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:36,648 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-18T23:17:36,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-18T23:17:36,652 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:36,652 DEBUG [Thread-2298 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T23:17:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,653 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:36,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9d96596e55a9,32899,1731971713394, seqNum=-1] 2024-11-18T23:17:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T23:17:36,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T23:17:36,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5] 2024-11-18T23:17:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971916690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971916693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971916695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731971856687, max timestamp=1731971856698Num uniques 859; 2024-11-18T23:17:36,701 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:36,703 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731971856687, max timestamp=1731971856698Num uniques 859; 2024-11-18T23:17:36,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:36,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731971856687, max timestamp=1731971856698Num uniques 859; 2024-11-18T23:17:36,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:36,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180a55dea8adbb453e9bbcbcbf1ff0f690_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971856639/Put/seqid=0 2024-11-18T23:17:36,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742333_1509 (size=12154) 2024-11-18T23:17:36,757 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-18T23:17:36,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971916767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971916770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,797 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180a55dea8adbb453e9bbcbcbf1ff0f690_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180a55dea8adbb453e9bbcbcbf1ff0f690_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:36,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-18T23:17:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:36,808 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/702611b7ad334eb1bb4882b0c8f94ea7, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:36,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/702611b7ad334eb1bb4882b0c8f94ea7 is 175, key is test_row_0/A:col10/1731971856639/Put/seqid=0 2024-11-18T23:17:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971916807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971916808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971916807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742334_1510 (size=30955) 2024-11-18T23:17:36,863 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/702611b7ad334eb1bb4882b0c8f94ea7 2024-11-18T23:17:36,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971916888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:36,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971916889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:36,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a is 50, key is test_row_0/B:col10/1731971856639/Put/seqid=0 2024-11-18T23:17:36,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-18T23:17:36,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-18T23:17:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:36,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:36,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:36,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742335_1511 (size=12001) 2024-11-18T23:17:37,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a 2024-11-18T23:17:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971917028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971917028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971917028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/4fba964966774b769c7e1dc0f773f2da is 50, key is test_row_0/C:col10/1731971856639/Put/seqid=0 2024-11-18T23:17:37,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742336_1512 (size=12001) 2024-11-18T23:17:37,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971917111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971917112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-18T23:17:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:37,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-18T23:17:37,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-18T23:17:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:37,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971917338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971917347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971917347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971917428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971917428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,434 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-18T23:17:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:37,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:37,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/4fba964966774b769c7e1dc0f773f2da 2024-11-18T23:17:37,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/702611b7ad334eb1bb4882b0c8f94ea7 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/702611b7ad334eb1bb4882b0c8f94ea7 2024-11-18T23:17:37,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/702611b7ad334eb1bb4882b0c8f94ea7, entries=150, sequenceid=15, filesize=30.2 K 2024-11-18T23:17:37,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a 2024-11-18T23:17:37,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T23:17:37,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/4fba964966774b769c7e1dc0f773f2da as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4fba964966774b769c7e1dc0f773f2da 2024-11-18T23:17:37,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4fba964966774b769c7e1dc0f773f2da, entries=150, sequenceid=15, filesize=11.7 K 2024-11-18T23:17:37,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for b28daae718a8c78c9acc9f3a87d1c727 in 891ms, sequenceid=15, compaction requested=false 2024-11-18T23:17:37,537 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-18T23:17:37,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:37,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-18T23:17:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:37,624 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-18T23:17:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:37,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:37,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:37,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:37,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:37,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111850fb734dbdf147438f3c3c78647af301_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971856735/Put/seqid=0 2024-11-18T23:17:37,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742337_1513 (size=12154) 2024-11-18T23:17:37,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:37,762 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111850fb734dbdf147438f3c3c78647af301_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111850fb734dbdf147438f3c3c78647af301_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:37,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/de659c5cd4c64758ab62eb35703acabf, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:37,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/de659c5cd4c64758ab62eb35703acabf is 175, key is test_row_0/A:col10/1731971856735/Put/seqid=0 2024-11-18T23:17:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-18T23:17:37,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742338_1514 (size=30955) 2024-11-18T23:17:37,856 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/de659c5cd4c64758ab62eb35703acabf 2024-11-18T23:17:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:37,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:37,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/7e2b529cbbfc4743807319cbce795a32 is 50, key is test_row_0/B:col10/1731971856735/Put/seqid=0 2024-11-18T23:17:37,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971917881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971917887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971917889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742339_1515 (size=12001) 2024-11-18T23:17:37,914 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/7e2b529cbbfc4743807319cbce795a32 2024-11-18T23:17:37,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/5b6f49b99dd047faa568464c0a113dcb is 50, key is test_row_0/C:col10/1731971856735/Put/seqid=0 2024-11-18T23:17:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971917940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971917947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:37,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742340_1516 (size=12001) 2024-11-18T23:17:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971917999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971918009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971918009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971918217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971918228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971918231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,397 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/5b6f49b99dd047faa568464c0a113dcb 2024-11-18T23:17:38,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/de659c5cd4c64758ab62eb35703acabf as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/de659c5cd4c64758ab62eb35703acabf 2024-11-18T23:17:38,422 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/de659c5cd4c64758ab62eb35703acabf, entries=150, sequenceid=50, filesize=30.2 K 2024-11-18T23:17:38,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/7e2b529cbbfc4743807319cbce795a32 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/7e2b529cbbfc4743807319cbce795a32 2024-11-18T23:17:38,428 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/7e2b529cbbfc4743807319cbce795a32, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T23:17:38,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/5b6f49b99dd047faa568464c0a113dcb as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/5b6f49b99dd047faa568464c0a113dcb 2024-11-18T23:17:38,432 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/5b6f49b99dd047faa568464c0a113dcb, entries=150, sequenceid=50, filesize=11.7 K 2024-11-18T23:17:38,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for b28daae718a8c78c9acc9f3a87d1c727 in 810ms, sequenceid=50, compaction requested=false 2024-11-18T23:17:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:38,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-18T23:17:38,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-18T23:17:38,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-18T23:17:38,436 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7810 sec 2024-11-18T23:17:38,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 1.7890 sec 2024-11-18T23:17:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:38,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-18T23:17:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:38,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:38,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111802f468bc83e84189a64d41bb8032d8e3_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971857871/Put/seqid=0 2024-11-18T23:17:38,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742341_1517 (size=12154) 2024-11-18T23:17:38,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:38,587 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111802f468bc83e84189a64d41bb8032d8e3_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111802f468bc83e84189a64d41bb8032d8e3_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:38,588 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/bd95ff50c69b4066a2b3bf0cc48f6cfb, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:38,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/bd95ff50c69b4066a2b3bf0cc48f6cfb is 175, key is test_row_0/A:col10/1731971857871/Put/seqid=0 2024-11-18T23:17:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971918581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,591 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971918582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971918585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742342_1518 (size=30955) 2024-11-18T23:17:38,618 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/bd95ff50c69b4066a2b3bf0cc48f6cfb 2024-11-18T23:17:38,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/6c297c09c4a64c179c92f0af91d871cd is 50, key is test_row_0/B:col10/1731971857871/Put/seqid=0 2024-11-18T23:17:38,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971918697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971918697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,701 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971918707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742343_1519 (size=12001) 2024-11-18T23:17:38,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-18T23:17:38,807 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:38,807 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:38,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-18T23:17:38,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T23:17:38,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:38,810 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:38,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:38,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T23:17:38,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971918907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971918908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971918918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-18T23:17:38,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:38,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:38,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:38,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:38,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:38,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971918960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,979 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_0' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:38,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971918977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:38,987 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:39,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/6c297c09c4a64c179c92f0af91d871cd 2024-11-18T23:17:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T23:17:39,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-18T23:17:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/8fc7d25b519e4ed9b150224febd7bb69 is 50, key is test_row_0/C:col10/1731971857871/Put/seqid=0 2024-11-18T23:17:39,213 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T23:17:39,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971919232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:39,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971919233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:39,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:39,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742344_1520 (size=12001) 2024-11-18T23:17:39,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971919244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:39,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-18T23:17:39,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:39,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T23:17:39,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-18T23:17:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-18T23:17:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/8fc7d25b519e4ed9b150224febd7bb69 2024-11-18T23:17:39,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/bd95ff50c69b4066a2b3bf0cc48f6cfb as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/bd95ff50c69b4066a2b3bf0cc48f6cfb 2024-11-18T23:17:39,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971919747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:39,756 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971919747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:39,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/bd95ff50c69b4066a2b3bf0cc48f6cfb, entries=150, sequenceid=63, filesize=30.2 K 2024-11-18T23:17:39,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/6c297c09c4a64c179c92f0af91d871cd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6c297c09c4a64c179c92f0af91d871cd 2024-11-18T23:17:39,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971919771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:39,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:39,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-18T23:17:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] handler.RSProcedureHandler(58): pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=141 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,806 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6c297c09c4a64c179c92f0af91d871cd, entries=150, sequenceid=63, filesize=11.7 K 2024-11-18T23:17:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=141 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:39,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/8fc7d25b519e4ed9b150224febd7bb69 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/8fc7d25b519e4ed9b150224febd7bb69 2024-11-18T23:17:39,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/8fc7d25b519e4ed9b150224febd7bb69, entries=150, sequenceid=63, filesize=11.7 K 2024-11-18T23:17:39,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for b28daae718a8c78c9acc9f3a87d1c727 in 1332ms, sequenceid=63, compaction requested=true 2024-11-18T23:17:39,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:39,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:39,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:39,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:39,861 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:39,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:39,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:39,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:39,861 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:39,867 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:39,867 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/B is initiating minor compaction (all files) 2024-11-18T23:17:39,867 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/B in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,867 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/7e2b529cbbfc4743807319cbce795a32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6c297c09c4a64c179c92f0af91d871cd] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=35.2 K 2024-11-18T23:17:39,873 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:39,874 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/A is initiating minor compaction (all files) 2024-11-18T23:17:39,874 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/A in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,874 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/702611b7ad334eb1bb4882b0c8f94ea7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/de659c5cd4c64758ab62eb35703acabf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/bd95ff50c69b4066a2b3bf0cc48f6cfb] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=90.7 K 2024-11-18T23:17:39,874 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,874 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/702611b7ad334eb1bb4882b0c8f94ea7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/de659c5cd4c64758ab62eb35703acabf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/bd95ff50c69b4066a2b3bf0cc48f6cfb] 2024-11-18T23:17:39,874 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c76f4c5dc9e04ae4bcc1abe86cd45b6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731971856627 2024-11-18T23:17:39,878 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702611b7ad334eb1bb4882b0c8f94ea7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731971856627 2024-11-18T23:17:39,878 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e2b529cbbfc4743807319cbce795a32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971856735 2024-11-18T23:17:39,879 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting de659c5cd4c64758ab62eb35703acabf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971856735 2024-11-18T23:17:39,879 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c297c09c4a64c179c92f0af91d871cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731971857871 2024-11-18T23:17:39,879 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd95ff50c69b4066a2b3bf0cc48f6cfb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731971857871 2024-11-18T23:17:39,925 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:39,937 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118e8c9858695de4fc7b678cea5ccd7ce9c_b28daae718a8c78c9acc9f3a87d1c727 store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:39,938 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:39,939 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/97dd0be6dd92426698375bde35b7da7b is 50, key is test_row_0/B:col10/1731971857871/Put/seqid=0 2024-11-18T23:17:39,939 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118e8c9858695de4fc7b678cea5ccd7ce9c_b28daae718a8c78c9acc9f3a87d1c727, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:39,940 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e8c9858695de4fc7b678cea5ccd7ce9c_b28daae718a8c78c9acc9f3a87d1c727 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T23:17:39,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-18T23:17:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:39,976 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T23:17:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:39,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:40,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742346_1522 (size=4469) 2024-11-18T23:17:40,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742345_1521 (size=12104) 2024-11-18T23:17:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a945988badd34f11a007166eb5915cfa_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971858580/Put/seqid=0 2024-11-18T23:17:40,062 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/97dd0be6dd92426698375bde35b7da7b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/97dd0be6dd92426698375bde35b7da7b 2024-11-18T23:17:40,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742347_1523 (size=12154) 2024-11-18T23:17:40,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,080 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/B of b28daae718a8c78c9acc9f3a87d1c727 into 97dd0be6dd92426698375bde35b7da7b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:40,080 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:40,080 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/B, priority=13, startTime=1731971859861; duration=0sec 2024-11-18T23:17:40,081 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:40,081 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:40,081 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:40,088 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:40,088 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/C is initiating minor compaction (all files) 2024-11-18T23:17:40,088 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/C in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:40,088 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4fba964966774b769c7e1dc0f773f2da, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/5b6f49b99dd047faa568464c0a113dcb, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/8fc7d25b519e4ed9b150224febd7bb69] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=35.2 K 2024-11-18T23:17:40,089 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fba964966774b769c7e1dc0f773f2da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731971856627 2024-11-18T23:17:40,090 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b6f49b99dd047faa568464c0a113dcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731971856735 2024-11-18T23:17:40,094 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118a945988badd34f11a007166eb5915cfa_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a945988badd34f11a007166eb5915cfa_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:40,094 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fc7d25b519e4ed9b150224febd7bb69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731971857871 2024-11-18T23:17:40,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:40,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279 is 175, key is test_row_0/A:col10/1731971858580/Put/seqid=0 2024-11-18T23:17:40,131 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:40,133 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/3f47627eb9f74356b94f55c0d5b245a3 is 50, key is test_row_0/C:col10/1731971857871/Put/seqid=0 2024-11-18T23:17:40,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742348_1524 (size=30955) 2024-11-18T23:17:40,145 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279 2024-11-18T23:17:40,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742349_1525 (size=12104) 2024-11-18T23:17:40,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/3655b95405d044febf467cd77262e423 is 50, key is test_row_0/B:col10/1731971858580/Put/seqid=0 2024-11-18T23:17:40,172 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/3f47627eb9f74356b94f55c0d5b245a3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/3f47627eb9f74356b94f55c0d5b245a3 2024-11-18T23:17:40,181 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/C of b28daae718a8c78c9acc9f3a87d1c727 into 3f47627eb9f74356b94f55c0d5b245a3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:40,181 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:40,181 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/C, priority=13, startTime=1731971859861; duration=0sec 2024-11-18T23:17:40,181 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:40,181 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:40,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742350_1526 (size=12001) 2024-11-18T23:17:40,212 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/3655b95405d044febf467cd77262e423 2024-11-18T23:17:40,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/15aeecf1f91d4f918788bb2de88aed58 is 50, key is test_row_0/C:col10/1731971858580/Put/seqid=0 2024-11-18T23:17:40,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742351_1527 (size=12001) 2024-11-18T23:17:40,237 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/15aeecf1f91d4f918788bb2de88aed58 2024-11-18T23:17:40,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279 2024-11-18T23:17:40,249 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279, entries=150, sequenceid=82, filesize=30.2 K 2024-11-18T23:17:40,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/3655b95405d044febf467cd77262e423 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3655b95405d044febf467cd77262e423 2024-11-18T23:17:40,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,258 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3655b95405d044febf467cd77262e423, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T23:17:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/15aeecf1f91d4f918788bb2de88aed58 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/15aeecf1f91d4f918788bb2de88aed58 2024-11-18T23:17:40,275 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/15aeecf1f91d4f918788bb2de88aed58, entries=150, sequenceid=82, filesize=11.7 K 2024-11-18T23:17:40,277 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for b28daae718a8c78c9acc9f3a87d1c727 in 301ms, sequenceid=82, compaction requested=false 2024-11-18T23:17:40,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:40,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:40,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-18T23:17:40,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-18T23:17:40,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-18T23:17:40,281 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4690 sec 2024-11-18T23:17:40,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.4750 sec 2024-11-18T23:17:40,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,417 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#A#compaction#443 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,417 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/b63a0c04e2ac45489ab0b67017bd7f5c is 175, key is test_row_0/A:col10/1731971857871/Put/seqid=0 2024-11-18T23:17:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742352_1528 (size=31058) 2024-11-18T23:17:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,458 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/b63a0c04e2ac45489ab0b67017bd7f5c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/b63a0c04e2ac45489ab0b67017bd7f5c 2024-11-18T23:17:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,468 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/A of b28daae718a8c78c9acc9f3a87d1c727 into b63a0c04e2ac45489ab0b67017bd7f5c(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:40,468 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,468 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/A, priority=13, startTime=1731971859861; duration=0sec 2024-11-18T23:17:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,468 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:40,468 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:40,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:17:40,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:40,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:40,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:40,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:40,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:40,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:40,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111842fda6666a0a4760bb813061c26ef5db_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971860899/Put/seqid=0 2024-11-18T23:17:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-18T23:17:40,978 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:40,979 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-18T23:17:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T23:17:40,984 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:40,985 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:40,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:40,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742353_1529 (size=24358) 2024-11-18T23:17:41,001 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:41,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971860979, max timestamp=1731971861000Num uniques 859; 2024-11-18T23:17:41,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971860979, max timestamp=1731971861000Num uniques 859; 2024-11-18T23:17:41,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:41,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:41,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971860979, max timestamp=1731971861000Num uniques 859; 2024-11-18T23:17:41,010 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:41,020 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111842fda6666a0a4760bb813061c26ef5db_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111842fda6666a0a4760bb813061c26ef5db_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:41,025 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/4ce1b93b9f944023a358759ba6ed9b64, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:41,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/4ce1b93b9f944023a358759ba6ed9b64 is 175, key is test_row_0/A:col10/1731971860899/Put/seqid=0 2024-11-18T23:17:41,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742354_1530 (size=73995) 2024-11-18T23:17:41,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/4ce1b93b9f944023a358759ba6ed9b64 2024-11-18T23:17:41,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/85ee7c821cb642f4b3018470e9edeb95 is 50, key is test_row_0/B:col10/1731971860899/Put/seqid=0 2024-11-18T23:17:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T23:17:41,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742355_1531 (size=12001) 2024-11-18T23:17:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971921118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971921125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,143 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:41,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971921130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971921132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971921144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971921247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,255 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971921260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,269 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971921267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T23:17:41,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971921292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971921292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,302 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:41,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,469 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-18T23:17:41,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971921457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971921478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971921487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/85ee7c821cb642f4b3018470e9edeb95 2024-11-18T23:17:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971921512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971921513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/bc2000ca7f024232b9c398e5cddbf9d4 is 50, key is test_row_0/C:col10/1731971860899/Put/seqid=0 2024-11-18T23:17:41,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742356_1532 (size=12001) 2024-11-18T23:17:41,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/bc2000ca7f024232b9c398e5cddbf9d4 2024-11-18T23:17:41,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T23:17:41,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/4ce1b93b9f944023a358759ba6ed9b64 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/4ce1b93b9f944023a358759ba6ed9b64 2024-11-18T23:17:41,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/4ce1b93b9f944023a358759ba6ed9b64, entries=400, sequenceid=96, filesize=72.3 K 2024-11-18T23:17:41,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/85ee7c821cb642f4b3018470e9edeb95 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/85ee7c821cb642f4b3018470e9edeb95 2024-11-18T23:17:41,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/85ee7c821cb642f4b3018470e9edeb95, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T23:17:41,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/bc2000ca7f024232b9c398e5cddbf9d4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/bc2000ca7f024232b9c398e5cddbf9d4 2024-11-18T23:17:41,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:41,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:41,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/bc2000ca7f024232b9c398e5cddbf9d4, entries=150, sequenceid=96, filesize=11.7 K 2024-11-18T23:17:41,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for b28daae718a8c78c9acc9f3a87d1c727 in 744ms, sequenceid=96, compaction requested=true 2024-11-18T23:17:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:41,649 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:41,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:41,649 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:41,650 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:41,651 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/B is initiating minor compaction (all files) 2024-11-18T23:17:41,651 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/B in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,651 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/97dd0be6dd92426698375bde35b7da7b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3655b95405d044febf467cd77262e423, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/85ee7c821cb642f4b3018470e9edeb95] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=35.3 K 2024-11-18T23:17:41,651 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 136008 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:41,651 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/A is initiating minor compaction (all files) 2024-11-18T23:17:41,652 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/A in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,652 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/b63a0c04e2ac45489ab0b67017bd7f5c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/4ce1b93b9f944023a358759ba6ed9b64] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=132.8 K 2024-11-18T23:17:41,652 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,652 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/b63a0c04e2ac45489ab0b67017bd7f5c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/4ce1b93b9f944023a358759ba6ed9b64] 2024-11-18T23:17:41,652 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 97dd0be6dd92426698375bde35b7da7b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731971857871 2024-11-18T23:17:41,652 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting b63a0c04e2ac45489ab0b67017bd7f5c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731971857871 2024-11-18T23:17:41,653 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3655b95405d044febf467cd77262e423, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731971858574 2024-11-18T23:17:41,653 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeb6da0c8b5f40f9ac8dc9c9f6e26279, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731971858574 2024-11-18T23:17:41,653 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 85ee7c821cb642f4b3018470e9edeb95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971860852 2024-11-18T23:17:41,654 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce1b93b9f944023a358759ba6ed9b64, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971860848 2024-11-18T23:17:41,666 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#B#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:41,667 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/23ee8e71eaef4f9e820de6345f9654a0 is 50, key is test_row_0/B:col10/1731971860899/Put/seqid=0 2024-11-18T23:17:41,671 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:41,675 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241118002e574cf20d4f7989dab6bc43ab05c9_b28daae718a8c78c9acc9f3a87d1c727 store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:41,679 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241118002e574cf20d4f7989dab6bc43ab05c9_b28daae718a8c78c9acc9f3a87d1c727, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:41,679 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118002e574cf20d4f7989dab6bc43ab05c9_b28daae718a8c78c9acc9f3a87d1c727 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:41,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742357_1533 (size=12207) 2024-11-18T23:17:41,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742358_1534 (size=4469) 2024-11-18T23:17:41,704 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#A#compaction#453 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:41,712 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ae4a25e5181d4bf28c8a716df216378e is 175, key is test_row_0/A:col10/1731971860899/Put/seqid=0 2024-11-18T23:17:41,723 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/23ee8e71eaef4f9e820de6345f9654a0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/23ee8e71eaef4f9e820de6345f9654a0 2024-11-18T23:17:41,738 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/B of b28daae718a8c78c9acc9f3a87d1c727 into 23ee8e71eaef4f9e820de6345f9654a0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:41,738 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:41,738 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/B, priority=13, startTime=1731971861649; duration=0sec 2024-11-18T23:17:41,738 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:41,738 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:41,739 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:41,740 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:41,740 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/C is initiating minor compaction (all files) 2024-11-18T23:17:41,740 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/C in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,740 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/3f47627eb9f74356b94f55c0d5b245a3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/15aeecf1f91d4f918788bb2de88aed58, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/bc2000ca7f024232b9c398e5cddbf9d4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=35.3 K 2024-11-18T23:17:41,740 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f47627eb9f74356b94f55c0d5b245a3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731971857871 2024-11-18T23:17:41,741 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 15aeecf1f91d4f918788bb2de88aed58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731971858574 2024-11-18T23:17:41,741 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting bc2000ca7f024232b9c398e5cddbf9d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971860852 2024-11-18T23:17:41,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742359_1535 (size=31161) 2024-11-18T23:17:41,791 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#C#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:41,792 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/835253ac3f434c3db2defbb448c4a94f is 50, key is test_row_0/C:col10/1731971860899/Put/seqid=0 2024-11-18T23:17:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:41,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-11-18T23:17:41,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:41,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:41,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:41,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:41,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:41,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:41,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:41,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:41,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118eaa301c821dc4b299a5b9f639ede7f3d_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971861067/Put/seqid=0 2024-11-18T23:17:41,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742360_1536 (size=12207) 2024-11-18T23:17:41,864 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/835253ac3f434c3db2defbb448c4a94f as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/835253ac3f434c3db2defbb448c4a94f 2024-11-18T23:17:41,870 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/C of b28daae718a8c78c9acc9f3a87d1c727 into 835253ac3f434c3db2defbb448c4a94f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:41,870 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:41,870 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/C, priority=13, startTime=1731971861649; duration=0sec 2024-11-18T23:17:41,870 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:41,871 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:41,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971921849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971921849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971921857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971921880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742361_1537 (size=14694) 2024-11-18T23:17:41,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:41,933 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118eaa301c821dc4b299a5b9f639ede7f3d_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118eaa301c821dc4b299a5b9f639ede7f3d_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:41,934 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7338320fd74e45c9ab08339a6f55c69d, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:41,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7338320fd74e45c9ab08339a6f55c69d is 175, key is test_row_0/A:col10/1731971861067/Put/seqid=0 2024-11-18T23:17:41,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:41,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971921902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:41,938 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:41,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:41,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:41,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:41,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742362_1538 (size=39649) 2024-11-18T23:17:42,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971922009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971922017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971922024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,031 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971922050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T23:17:42,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:42,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,189 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ae4a25e5181d4bf28c8a716df216378e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae4a25e5181d4bf28c8a716df216378e 2024-11-18T23:17:42,199 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/A of b28daae718a8c78c9acc9f3a87d1c727 into ae4a25e5181d4bf28c8a716df216378e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:42,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:42,199 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/A, priority=13, startTime=1731971861649; duration=0sec 2024-11-18T23:17:42,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:42,199 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971922227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971922238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971922240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,248 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971922281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:42,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:42,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:42,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:42,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,416 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7338320fd74e45c9ab08339a6f55c69d 2024-11-18T23:17:42,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/53e13288516140249010edfe8dd3328e is 50, key is test_row_0/B:col10/1731971861067/Put/seqid=0 2024-11-18T23:17:42,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971922428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742363_1539 (size=12101) 2024-11-18T23:17:42,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:42,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:42,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:42,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:42,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:42,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/53e13288516140249010edfe8dd3328e 2024-11-18T23:17:42,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/9da7221e4e1f4b18a528eb26d106e5cd is 50, key is test_row_0/C:col10/1731971861067/Put/seqid=0 2024-11-18T23:17:42,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971922547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742364_1540 (size=12101) 2024-11-18T23:17:42,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/9da7221e4e1f4b18a528eb26d106e5cd 2024-11-18T23:17:42,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971922557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,565 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971922557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7338320fd74e45c9ab08339a6f55c69d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7338320fd74e45c9ab08339a6f55c69d 2024-11-18T23:17:42,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7338320fd74e45c9ab08339a6f55c69d, entries=200, sequenceid=134, filesize=38.7 K 2024-11-18T23:17:42,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/53e13288516140249010edfe8dd3328e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/53e13288516140249010edfe8dd3328e 2024-11-18T23:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/53e13288516140249010edfe8dd3328e, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T23:17:42,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/9da7221e4e1f4b18a528eb26d106e5cd as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/9da7221e4e1f4b18a528eb26d106e5cd 2024-11-18T23:17:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971922597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:42,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:42,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/9da7221e4e1f4b18a528eb26d106e5cd, entries=150, sequenceid=134, filesize=11.8 K 2024-11-18T23:17:42,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=40.25 KB/41220 for b28daae718a8c78c9acc9f3a87d1c727 in 810ms, sequenceid=134, compaction requested=false 2024-11-18T23:17:42,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-18T23:17:42,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:42,647 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-18T23:17:42,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:42,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:42,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:42,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:42,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:42,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:42,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e3f4c3ea062a493a8f722794383c68af_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_1/A:col10/1731971861845/Put/seqid=0 2024-11-18T23:17:42,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742365_1541 (size=9814) 2024-11-18T23:17:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,712 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118e3f4c3ea062a493a8f722794383c68af_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e3f4c3ea062a493a8f722794383c68af_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/f534249813bc4af8bec1c6c1cbd923e0, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/f534249813bc4af8bec1c6c1cbd923e0 is 175, key is test_row_1/A:col10/1731971861845/Put/seqid=0 2024-11-18T23:17:42,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742366_1542 (size=22461) 2024-11-18T23:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:42,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:43,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T23:17:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:43,150 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=145, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/f534249813bc4af8bec1c6c1cbd923e0 2024-11-18T23:17:43,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/93322b7ee4014b6181051ce435378f83 is 50, key is test_row_1/B:col10/1731971861845/Put/seqid=0 2024-11-18T23:17:43,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971863180, max timestamp=1731971863203Num uniques 859; 2024-11-18T23:17:43,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:43,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971863180, max timestamp=1731971863203Num uniques 859; 2024-11-18T23:17:43,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:43,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971863180, max timestamp=1731971863203Num uniques 859; 2024-11-18T23:17:43,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:43,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742367_1543 (size=9757) 2024-11-18T23:17:43,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731971863244, max timestamp=1731971863278Num uniques 1018; 2024-11-18T23:17:43,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-18T23:17:43,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731971863247, max timestamp=1731971863278Num uniques 1018; 2024-11-18T23:17:43,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T23:17:43,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731971863247, max timestamp=1731971863278Num uniques 1018; 2024-11-18T23:17:43,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-18T23:17:43,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971923350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971923351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971923353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971923359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,369 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971923448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,453 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2322 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971923471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,477 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971923471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971923471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971923479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/93322b7ee4014b6181051ce435378f83 2024-11-18T23:17:43,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/769eb3113b6744879611b575436291c8 is 50, key is test_row_1/C:col10/1731971861845/Put/seqid=0 2024-11-18T23:17:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971923690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971923690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971923691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971923700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:43,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:43,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742368_1544 (size=9757) 2024-11-18T23:17:44,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971923998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971924009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971924009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971924021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,027 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,115 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/769eb3113b6744879611b575436291c8 2024-11-18T23:17:44,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/f534249813bc4af8bec1c6c1cbd923e0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f534249813bc4af8bec1c6c1cbd923e0 2024-11-18T23:17:44,155 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f534249813bc4af8bec1c6c1cbd923e0, entries=100, sequenceid=145, filesize=21.9 K 2024-11-18T23:17:44,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/93322b7ee4014b6181051ce435378f83 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93322b7ee4014b6181051ce435378f83 2024-11-18T23:17:44,165 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93322b7ee4014b6181051ce435378f83, entries=100, sequenceid=145, filesize=9.5 K 2024-11-18T23:17:44,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/769eb3113b6744879611b575436291c8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/769eb3113b6744879611b575436291c8 2024-11-18T23:17:44,181 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/769eb3113b6744879611b575436291c8, entries=100, sequenceid=145, filesize=9.5 K 2024-11-18T23:17:44,182 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for b28daae718a8c78c9acc9f3a87d1c727 in 1536ms, sequenceid=145, compaction requested=true 2024-11-18T23:17:44,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:44,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:44,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-18T23:17:44,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-18T23:17:44,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-18T23:17:44,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2780 sec 2024-11-18T23:17:44,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 3.2900 sec 2024-11-18T23:17:44,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-18T23:17:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:17:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:17:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:44,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-18T23:17:44,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:44,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111893391db7c23b49de9b018525ded4d7d9_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971863336/Put/seqid=0 2024-11-18T23:17:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971924541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971924549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,551 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971924552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971924552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742369_1545 (size=14794) 2024-11-18T23:17:44,597 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:44,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111893391db7c23b49de9b018525ded4d7d9_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111893391db7c23b49de9b018525ded4d7d9_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:44,603 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/02060da2cf964cbdb0ddc5e984105d3a, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:44,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/02060da2cf964cbdb0ddc5e984105d3a is 175, key is test_row_0/A:col10/1731971863336/Put/seqid=0 2024-11-18T23:17:44,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742370_1546 (size=39749) 2024-11-18T23:17:44,622 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=200, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/02060da2cf964cbdb0ddc5e984105d3a 2024-11-18T23:17:44,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/15a98214802149839d77fb0a913033b4 is 50, key is test_row_0/B:col10/1731971863336/Put/seqid=0 2024-11-18T23:17:44,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742371_1547 (size=12151) 2024-11-18T23:17:44,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/15a98214802149839d77fb0a913033b4 2024-11-18T23:17:44,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/64e28894db1040e1ab59f62540c64e17 is 50, key is test_row_0/C:col10/1731971863336/Put/seqid=0 2024-11-18T23:17:44,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971924657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971924657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742372_1548 (size=12151) 2024-11-18T23:17:44,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971924681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971924682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971924868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971924868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971924899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971924899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,904 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:44,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/64e28894db1040e1ab59f62540c64e17 2024-11-18T23:17:45,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/02060da2cf964cbdb0ddc5e984105d3a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/02060da2cf964cbdb0ddc5e984105d3a 2024-11-18T23:17:45,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/02060da2cf964cbdb0ddc5e984105d3a, entries=200, sequenceid=200, filesize=38.8 K 2024-11-18T23:17:45,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/15a98214802149839d77fb0a913033b4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/15a98214802149839d77fb0a913033b4 2024-11-18T23:17:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/15a98214802149839d77fb0a913033b4, entries=150, sequenceid=200, filesize=11.9 K 2024-11-18T23:17:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/64e28894db1040e1ab59f62540c64e17 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/64e28894db1040e1ab59f62540c64e17 2024-11-18T23:17:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/64e28894db1040e1ab59f62540c64e17, entries=150, sequenceid=200, filesize=11.9 K 2024-11-18T23:17:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for b28daae718a8c78c9acc9f3a87d1c727 in 568ms, sequenceid=200, compaction requested=true 2024-11-18T23:17:45,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,097 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:45,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:45,098 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:45,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:45,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:45,100 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133020 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:45,100 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/A is initiating minor compaction (all files) 2024-11-18T23:17:45,100 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/A in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,100 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae4a25e5181d4bf28c8a716df216378e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7338320fd74e45c9ab08339a6f55c69d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f534249813bc4af8bec1c6c1cbd923e0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/02060da2cf964cbdb0ddc5e984105d3a] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=129.9 K 2024-11-18T23:17:45,100 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,100 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae4a25e5181d4bf28c8a716df216378e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7338320fd74e45c9ab08339a6f55c69d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f534249813bc4af8bec1c6c1cbd923e0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/02060da2cf964cbdb0ddc5e984105d3a] 2024-11-18T23:17:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,101 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:45,101 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae4a25e5181d4bf28c8a716df216378e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971860852 2024-11-18T23:17:45,101 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/B is initiating minor compaction (all files) 2024-11-18T23:17:45,101 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/B in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,101 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/23ee8e71eaef4f9e820de6345f9654a0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/53e13288516140249010edfe8dd3328e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93322b7ee4014b6181051ce435378f83, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/15a98214802149839d77fb0a913033b4] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=45.1 K 2024-11-18T23:17:45,101 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7338320fd74e45c9ab08339a6f55c69d, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731971861025 2024-11-18T23:17:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,101 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting f534249813bc4af8bec1c6c1cbd923e0, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971861845 2024-11-18T23:17:45,101 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ee8e71eaef4f9e820de6345f9654a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971860852 2024-11-18T23:17:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,102 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02060da2cf964cbdb0ddc5e984105d3a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731971863336 2024-11-18T23:17:45,102 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e13288516140249010edfe8dd3328e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731971861067 2024-11-18T23:17:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,102 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 93322b7ee4014b6181051ce435378f83, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971861845 2024-11-18T23:17:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,103 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a98214802149839d77fb0a913033b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731971863336 2024-11-18T23:17:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,110 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,113 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,113 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/93f4d9fc1c974b949d7ac44b8aaefd60 is 50, key is test_row_0/B:col10/1731971863336/Put/seqid=0 2024-11-18T23:17:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,114 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111806035c949f084b5e8f803269b50211ec_b28daae718a8c78c9acc9f3a87d1c727 store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,118 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111806035c949f084b5e8f803269b50211ec_b28daae718a8c78c9acc9f3a87d1c727, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,118 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111806035c949f084b5e8f803269b50211ec_b28daae718a8c78c9acc9f3a87d1c727 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742373_1549 (size=12493) 2024-11-18T23:17:45,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742374_1550 (size=4469) 2024-11-18T23:17:45,128 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/93f4d9fc1c974b949d7ac44b8aaefd60 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93f4d9fc1c974b949d7ac44b8aaefd60 2024-11-18T23:17:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,132 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/B of b28daae718a8c78c9acc9f3a87d1c727 into 93f4d9fc1c974b949d7ac44b8aaefd60(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:45,132 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:45,132 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/B, priority=12, startTime=1731971865097; duration=0sec 2024-11-18T23:17:45,132 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:45,132 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,132 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,133 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46216 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:45,133 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/C is initiating minor compaction (all files) 2024-11-18T23:17:45,133 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/C in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,133 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/835253ac3f434c3db2defbb448c4a94f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/9da7221e4e1f4b18a528eb26d106e5cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/769eb3113b6744879611b575436291c8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/64e28894db1040e1ab59f62540c64e17] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=45.1 K 2024-11-18T23:17:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,133 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 835253ac3f434c3db2defbb448c4a94f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731971860852 2024-11-18T23:17:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,133 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da7221e4e1f4b18a528eb26d106e5cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731971861067 2024-11-18T23:17:45,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,134 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 769eb3113b6744879611b575436291c8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731971861845 2024-11-18T23:17:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,134 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e28894db1040e1ab59f62540c64e17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731971863336 2024-11-18T23:17:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,141 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#C#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,141 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/4beea62834af47d49732447a25797c8b is 50, key is test_row_0/C:col10/1731971863336/Put/seqid=0 2024-11-18T23:17:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742375_1551 (size=12493) 2024-11-18T23:17:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,148 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/4beea62834af47d49732447a25797c8b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4beea62834af47d49732447a25797c8b 2024-11-18T23:17:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,152 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/C of b28daae718a8c78c9acc9f3a87d1c727 into 4beea62834af47d49732447a25797c8b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:45,152 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,152 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/C, priority=12, startTime=1731971865100; duration=0sec 2024-11-18T23:17:45,152 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:45,152 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-18T23:17:45,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,157 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:45,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-18T23:17:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,160 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-18T23:17:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,161 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:45,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:45,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:17:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:45,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:45,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188fcfb3798501478c826e0a79d749ee76_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_1/A:col10/1731971865180/Put/seqid=0 2024-11-18T23:17:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742376_1552 (size=14794) 2024-11-18T23:17:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,204 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:45,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188fcfb3798501478c826e0a79d749ee76_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fcfb3798501478c826e0a79d749ee76_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:45,236 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ad7e79291aad471a995cd9b125f8b3ce, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:45,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ad7e79291aad471a995cd9b125f8b3ce is 175, key is test_row_1/A:col10/1731971865180/Put/seqid=0 2024-11-18T23:17:45,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971925242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742377_1553 (size=39745) 2024-11-18T23:17:45,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971925243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971925245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971925245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,251 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,256 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ad7e79291aad471a995cd9b125f8b3ce 2024-11-18T23:17:45,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/6cf7be076f5346308363faa5092223f9 is 50, key is test_row_1/B:col10/1731971865180/Put/seqid=0 2024-11-18T23:17:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-18T23:17:45,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742378_1554 (size=9757) 2024-11-18T23:17:45,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/6cf7be076f5346308363faa5092223f9 2024-11-18T23:17:45,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-18T23:17:45,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:45,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/ffe3ed010eee4cf3a3bf964b753fbc9a is 50, key is test_row_1/C:col10/1731971865180/Put/seqid=0 2024-11-18T23:17:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971925357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971925358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971925360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971925360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742379_1555 (size=9757) 2024-11-18T23:17:45,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-18T23:17:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971925477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,479 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4347 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-18T23:17:45,530 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#A#compaction#464 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:45,531 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/09f7eaf8105c423db96a95b89aae8b2c is 175, key is test_row_0/A:col10/1731971863336/Put/seqid=0 2024-11-18T23:17:45,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742380_1556 (size=31447) 2024-11-18T23:17:45,560 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/09f7eaf8105c423db96a95b89aae8b2c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/09f7eaf8105c423db96a95b89aae8b2c 2024-11-18T23:17:45,566 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/A of b28daae718a8c78c9acc9f3a87d1c727 into 09f7eaf8105c423db96a95b89aae8b2c(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:45,566 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:45,566 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/A, priority=12, startTime=1731971865097; duration=0sec 2024-11-18T23:17:45,566 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:45,566 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:45,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971925567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971925567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,568 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971925577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971925578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-18T23:17:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/ffe3ed010eee4cf3a3bf964b753fbc9a 2024-11-18T23:17:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-18T23:17:45,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-18T23:17:45,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:45,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:45,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ad7e79291aad471a995cd9b125f8b3ce as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ad7e79291aad471a995cd9b125f8b3ce 2024-11-18T23:17:45,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ad7e79291aad471a995cd9b125f8b3ce, entries=200, sequenceid=212, filesize=38.8 K 2024-11-18T23:17:45,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/6cf7be076f5346308363faa5092223f9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6cf7be076f5346308363faa5092223f9 2024-11-18T23:17:45,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971925877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,880 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6cf7be076f5346308363faa5092223f9, entries=100, sequenceid=212, filesize=9.5 K 2024-11-18T23:17:45,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/ffe3ed010eee4cf3a3bf964b753fbc9a as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ffe3ed010eee4cf3a3bf964b753fbc9a 2024-11-18T23:17:45,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971925880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971925888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971925888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:45,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ffe3ed010eee4cf3a3bf964b753fbc9a, entries=100, sequenceid=212, filesize=9.5 K 2024-11-18T23:17:45,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for b28daae718a8c78c9acc9f3a87d1c727 in 748ms, sequenceid=212, compaction requested=false 2024-11-18T23:17:45,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:45,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-18T23:17:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:45,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-18T23:17:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:46,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181d6d76518b054fec88b7474c090ac561_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971865243/Put/seqid=0 2024-11-18T23:17:46,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742381_1557 (size=12304) 2024-11-18T23:17:46,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:46,086 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181d6d76518b054fec88b7474c090ac561_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181d6d76518b054fec88b7474c090ac561_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:46,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/d29dd2c3af0841279a35fcb67644cc50, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:46,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/d29dd2c3af0841279a35fcb67644cc50 is 175, key is test_row_0/A:col10/1731971865243/Put/seqid=0 2024-11-18T23:17:46,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742382_1558 (size=31105) 2024-11-18T23:17:46,148 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/d29dd2c3af0841279a35fcb67644cc50 2024-11-18T23:17:46,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/71cef26f5db742bebdae60d25eec1204 is 50, key is test_row_0/B:col10/1731971865243/Put/seqid=0 2024-11-18T23:17:46,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742383_1559 (size=12151) 2024-11-18T23:17:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-18T23:17:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:46,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971926416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971926416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971926425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971926432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971926537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971926538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971926541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971926548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/71cef26f5db742bebdae60d25eec1204 2024-11-18T23:17:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/f1e693eb9377448aacd585ba835d86f2 is 50, key is test_row_0/C:col10/1731971865243/Put/seqid=0 2024-11-18T23:17:46,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742384_1560 (size=12151) 2024-11-18T23:17:46,736 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/f1e693eb9377448aacd585ba835d86f2 2024-11-18T23:17:46,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/d29dd2c3af0841279a35fcb67644cc50 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/d29dd2c3af0841279a35fcb67644cc50 2024-11-18T23:17:46,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971926747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,755 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971926757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971926760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,771 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/d29dd2c3af0841279a35fcb67644cc50, entries=150, sequenceid=235, filesize=30.4 K 2024-11-18T23:17:46,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/71cef26f5db742bebdae60d25eec1204 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/71cef26f5db742bebdae60d25eec1204 2024-11-18T23:17:46,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:46,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971926772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:46,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:46,787 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/71cef26f5db742bebdae60d25eec1204, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T23:17:46,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/f1e693eb9377448aacd585ba835d86f2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f1e693eb9377448aacd585ba835d86f2 2024-11-18T23:17:46,810 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f1e693eb9377448aacd585ba835d86f2, entries=150, sequenceid=235, filesize=11.9 K 2024-11-18T23:17:46,815 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=53.67 KB/54960 for b28daae718a8c78c9acc9f3a87d1c727 in 843ms, sequenceid=235, compaction requested=true 2024-11-18T23:17:46,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-18T23:17:46,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-18T23:17:46,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.6670 sec 2024-11-18T23:17:46,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-18T23:17:46,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6570 sec 2024-11-18T23:17:47,066 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-18T23:17:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:47,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:47,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186cd0da578efe4113bad14c16abfd44c4_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971866413/Put/seqid=0 2024-11-18T23:17:47,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742385_1561 (size=14794) 2024-11-18T23:17:47,124 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:47,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971927122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971927122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411186cd0da578efe4113bad14c16abfd44c4_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186cd0da578efe4113bad14c16abfd44c4_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7783f5a8f6b94dfe94711c8bb72a7ecc, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:47,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7783f5a8f6b94dfe94711c8bb72a7ecc is 175, key is test_row_0/A:col10/1731971866413/Put/seqid=0 2024-11-18T23:17:47,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742386_1562 (size=39749) 2024-11-18T23:17:47,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971927126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971927128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971927238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971927238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971927238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971927247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,252 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-18T23:17:47,320 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:47,320 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-18T23:17:47,323 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T23:17:47,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:47,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T23:17:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971927447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971927447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971927447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,451 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971927457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:47,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:47,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:47,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:47,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,537 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7783f5a8f6b94dfe94711c8bb72a7ecc 2024-11-18T23:17:47,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f1e3c1bf7cd64c61b55758ab903aa5e2 is 50, key is test_row_0/B:col10/1731971866413/Put/seqid=0 2024-11-18T23:17:47,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742387_1563 (size=12151) 2024-11-18T23:17:47,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f1e3c1bf7cd64c61b55758ab903aa5e2 2024-11-18T23:17:47,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/c1aef64ecaac49e5928c2ca523e4ddf2 is 50, key is test_row_0/C:col10/1731971866413/Put/seqid=0 2024-11-18T23:17:47,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:47,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T23:17:47,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742388_1564 (size=12151) 2024-11-18T23:17:47,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/c1aef64ecaac49e5928c2ca523e4ddf2 2024-11-18T23:17:47,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/7783f5a8f6b94dfe94711c8bb72a7ecc as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7783f5a8f6b94dfe94711c8bb72a7ecc 2024-11-18T23:17:47,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7783f5a8f6b94dfe94711c8bb72a7ecc, entries=200, sequenceid=247, filesize=38.8 K 2024-11-18T23:17:47,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f1e3c1bf7cd64c61b55758ab903aa5e2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f1e3c1bf7cd64c61b55758ab903aa5e2 2024-11-18T23:17:47,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f1e3c1bf7cd64c61b55758ab903aa5e2, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T23:17:47,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/c1aef64ecaac49e5928c2ca523e4ddf2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c1aef64ecaac49e5928c2ca523e4ddf2 2024-11-18T23:17:47,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c1aef64ecaac49e5928c2ca523e4ddf2, entries=150, sequenceid=247, filesize=11.9 K 2024-11-18T23:17:47,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for b28daae718a8c78c9acc9f3a87d1c727 in 621ms, sequenceid=247, compaction requested=true 2024-11-18T23:17:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:47,687 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:47,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:47,688 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:47,696 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:47,697 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/B is initiating minor compaction (all files) 2024-11-18T23:17:47,697 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/B in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,697 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93f4d9fc1c974b949d7ac44b8aaefd60, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6cf7be076f5346308363faa5092223f9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/71cef26f5db742bebdae60d25eec1204, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f1e3c1bf7cd64c61b55758ab903aa5e2] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=45.5 K 2024-11-18T23:17:47,700 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142046 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:47,700 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/A is initiating minor compaction (all files) 2024-11-18T23:17:47,700 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/A in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,701 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/09f7eaf8105c423db96a95b89aae8b2c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ad7e79291aad471a995cd9b125f8b3ce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/d29dd2c3af0841279a35fcb67644cc50, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7783f5a8f6b94dfe94711c8bb72a7ecc] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=138.7 K 2024-11-18T23:17:47,701 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,701 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/09f7eaf8105c423db96a95b89aae8b2c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ad7e79291aad471a995cd9b125f8b3ce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/d29dd2c3af0841279a35fcb67644cc50, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7783f5a8f6b94dfe94711c8bb72a7ecc] 2024-11-18T23:17:47,702 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 93f4d9fc1c974b949d7ac44b8aaefd60, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731971863336 2024-11-18T23:17:47,704 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f7eaf8105c423db96a95b89aae8b2c, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731971863336 2024-11-18T23:17:47,706 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cf7be076f5346308363faa5092223f9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731971865180 2024-11-18T23:17:47,706 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7e79291aad471a995cd9b125f8b3ce, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731971864536 2024-11-18T23:17:47,708 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 71cef26f5db742bebdae60d25eec1204, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731971865237 2024-11-18T23:17:47,708 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting d29dd2c3af0841279a35fcb67644cc50, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731971865237 2024-11-18T23:17:47,709 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e3c1bf7cd64c61b55758ab903aa5e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971866413 2024-11-18T23:17:47,709 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7783f5a8f6b94dfe94711c8bb72a7ecc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971866413 2024-11-18T23:17:47,737 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:47,743 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:47,743 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f43168a7acbd43caa6f504c2e138fde0 is 50, key is test_row_0/B:col10/1731971866413/Put/seqid=0 2024-11-18T23:17:47,753 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111866553719b70041e391adeebaaa881f6d_b28daae718a8c78c9acc9f3a87d1c727 store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:47,756 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111866553719b70041e391adeebaaa881f6d_b28daae718a8c78c9acc9f3a87d1c727, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:47,756 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866553719b70041e391adeebaaa881f6d_b28daae718a8c78c9acc9f3a87d1c727 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:47,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971867106, max timestamp=1731971867768Num uniques 859; 2024-11-18T23:17:47,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:47,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971867106, max timestamp=1731971867770Num uniques 859; 2024-11-18T23:17:47,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:47,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971867106, max timestamp=1731971867770Num uniques 859; 2024-11-18T23:17:47,779 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:47,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=33.54 KB heapSize=86.16 KB 2024-11-18T23:17:47,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:47,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:47,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:47,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:47,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:47,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:47,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:47,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:47,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742390_1566 (size=4469) 2024-11-18T23:17:47,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742389_1565 (size=12629) 2024-11-18T23:17:47,809 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#A#compaction#476 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:47,818 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/48bfd3946f9f4da59fb03f75c8890a11 is 175, key is test_row_0/A:col10/1731971866413/Put/seqid=0 2024-11-18T23:17:47,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ed2e6e2edc5844ac9084e8d0e2a3bf3f_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971867779/Put/seqid=0 2024-11-18T23:17:47,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742391_1567 (size=31583) 2024-11-18T23:17:47,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742392_1568 (size=12454) 2024-11-18T23:17:47,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:47,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731971867859, max timestamp=1731971867881Num uniques 859; 2024-11-18T23:17:47,887 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-18T23:17:47,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971867859, max timestamp=1731971867881Num uniques 859; 2024-11-18T23:17:47,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971867859, max timestamp=1731971867881Num uniques 859; 2024-11-18T23:17:47,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:17:47,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:17:47,893 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118ed2e6e2edc5844ac9084e8d0e2a3bf3f_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ed2e6e2edc5844ac9084e8d0e2a3bf3f_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:47,904 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/782fb73a6f8d4bf88c43e8f1558f1759, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:47,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/782fb73a6f8d4bf88c43e8f1558f1759 is 175, key is test_row_0/A:col10/1731971867779/Put/seqid=0 2024-11-18T23:17:47,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742393_1569 (size=31255) 2024-11-18T23:17:47,944 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/782fb73a6f8d4bf88c43e8f1558f1759 2024-11-18T23:17:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T23:17:47,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731971867932, max timestamp=1731971867962Num uniques 1068; 2024-11-18T23:17:47,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731971867932, max timestamp=1731971867962Num uniques 1009; 2024-11-18T23:17:47,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731971867932, max timestamp=1731971867962Num uniques 1009; 2024-11-18T23:17:47,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-18T23:17:47,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-18T23:17:47,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-18T23:17:47,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:47,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:47,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:47,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/ba3842927d3f486181eb7ba972f5b170 is 50, key is test_row_0/B:col10/1731971867779/Put/seqid=0 2024-11-18T23:17:48,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742394_1570 (size=12301) 2024-11-18T23:17:48,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971868050, max timestamp=1731971868092Num uniques 1077; 2024-11-18T23:17:48,100 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T23:17:48,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971868050, max timestamp=1731971868092Num uniques 1077; 2024-11-18T23:17:48,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T23:17:48,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731971868050, max timestamp=1731971868092Num uniques 1077; 2024-11-18T23:17:48,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-18T23:17:48,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:48,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731971868165, max timestamp=1731971868197Num uniques 1086; 2024-11-18T23:17:48,201 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-18T23:17:48,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971868165, max timestamp=1731971868197Num uniques 1086; 2024-11-18T23:17:48,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-11-18T23:17:48,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971868165, max timestamp=1731971868197Num uniques 1086; 2024-11-18T23:17:48,211 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=6870, suffix heap size=140912,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=150 2024-11-18T23:17:48,231 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f43168a7acbd43caa6f504c2e138fde0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f43168a7acbd43caa6f504c2e138fde0 2024-11-18T23:17:48,272 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/48bfd3946f9f4da59fb03f75c8890a11 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/48bfd3946f9f4da59fb03f75c8890a11 2024-11-18T23:17:48,296 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/B of b28daae718a8c78c9acc9f3a87d1c727 into f43168a7acbd43caa6f504c2e138fde0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:48,297 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:48,297 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/B, priority=12, startTime=1731971867687; duration=0sec 2024-11-18T23:17:48,297 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:48,297 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:48,297 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:48,297 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/A of b28daae718a8c78c9acc9f3a87d1c727 into 48bfd3946f9f4da59fb03f75c8890a11(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:48,297 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:48,297 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/A, priority=12, startTime=1731971867687; duration=0sec 2024-11-18T23:17:48,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:48,298 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:48,298 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,315 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:48,315 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/C is initiating minor compaction (all files) 2024-11-18T23:17:48,315 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/C in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,316 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4beea62834af47d49732447a25797c8b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ffe3ed010eee4cf3a3bf964b753fbc9a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f1e693eb9377448aacd585ba835d86f2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c1aef64ecaac49e5928c2ca523e4ddf2] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=45.5 K 2024-11-18T23:17:48,320 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 4beea62834af47d49732447a25797c8b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731971863336 2024-11-18T23:17:48,324 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe3ed010eee4cf3a3bf964b753fbc9a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731971865180 2024-11-18T23:17:48,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731971868214, max timestamp=1731971868318Num uniques 1045; 2024-11-18T23:17:48,324 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-18T23:17:48,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731971868214, max timestamp=1731971868318Num uniques 1009; 2024-11-18T23:17:48,325 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-18T23:17:48,328 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f1e693eb9377448aacd585ba835d86f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731971865237 2024-11-18T23:17:48,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731971868214, max timestamp=1731971868318Num uniques 1009; 2024-11-18T23:17:48,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11220, suffix heap size=132904,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=245 2024-11-18T23:17:48,329 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c1aef64ecaac49e5928c2ca523e4ddf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971866413 2024-11-18T23:17:48,382 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#C#compaction#480 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:48,382 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/b59384bc28eb4ab481ee1d079e629646 is 50, key is test_row_0/C:col10/1731971866413/Put/seqid=0 2024-11-18T23:17:48,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742395_1571 (size=12629) 2024-11-18T23:17:48,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971928409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971928418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971928419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971928422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,430 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,431 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/b59384bc28eb4ab481ee1d079e629646 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b59384bc28eb4ab481ee1d079e629646 2024-11-18T23:17:48,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/ba3842927d3f486181eb7ba972f5b170 2024-11-18T23:17:48,440 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/C of b28daae718a8c78c9acc9f3a87d1c727 into b59384bc28eb4ab481ee1d079e629646(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:48,440 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:48,440 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/C, priority=12, startTime=1731971867687; duration=0sec 2024-11-18T23:17:48,441 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:48,441 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:48,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/02fc5841bbdc4de78969e0486865935d is 50, key is test_row_0/C:col10/1731971867779/Put/seqid=0 2024-11-18T23:17:48,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T23:17:48,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742396_1572 (size=12301) 2024-11-18T23:17:48,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/02fc5841bbdc4de78969e0486865935d 2024-11-18T23:17:48,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/782fb73a6f8d4bf88c43e8f1558f1759 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/782fb73a6f8d4bf88c43e8f1558f1759 2024-11-18T23:17:48,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971928527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971928538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971928538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/782fb73a6f8d4bf88c43e8f1558f1759, entries=150, sequenceid=270, filesize=30.5 K 2024-11-18T23:17:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/ba3842927d3f486181eb7ba972f5b170 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ba3842927d3f486181eb7ba972f5b170 2024-11-18T23:17:48,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971928544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ba3842927d3f486181eb7ba972f5b170, entries=150, sequenceid=270, filesize=12.0 K 2024-11-18T23:17:48,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/02fc5841bbdc4de78969e0486865935d as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/02fc5841bbdc4de78969e0486865935d 2024-11-18T23:17:48,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/02fc5841bbdc4de78969e0486865935d, entries=150, sequenceid=270, filesize=12.0 K 2024-11-18T23:17:48,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=147.60 KB/151140 for b28daae718a8c78c9acc9f3a87d1c727 in 805ms, sequenceid=270, compaction requested=false 2024-11-18T23:17:48,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:48,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-18T23:17:48,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:48,612 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=147.60 KB heapSize=416.77 KB 2024-11-18T23:17:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:48,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111808fd491c041e415db4e5699fac2dfbf4_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971868378/Put/seqid=0 2024-11-18T23:17:48,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742397_1573 (size=12454) 2024-11-18T23:17:48,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:48,701 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111808fd491c041e415db4e5699fac2dfbf4_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111808fd491c041e415db4e5699fac2dfbf4_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/691cf79d72b54e899fba281e4a9f12ec, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/691cf79d72b54e899fba281e4a9f12ec is 175, key is test_row_0/A:col10/1731971868378/Put/seqid=0 2024-11-18T23:17:48,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742398_1574 (size=31255) 2024-11-18T23:17:48,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:48,743 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/691cf79d72b54e899fba281e4a9f12ec 2024-11-18T23:17:48,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971928757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971928769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971928772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971928777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/b101afc5162b47909ec7a340d11f4309 is 50, key is test_row_0/B:col10/1731971868378/Put/seqid=0 2024-11-18T23:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742399_1575 (size=12301) 2024-11-18T23:17:48,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971928887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971928900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:48,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971928900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:48,908 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971929077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971929112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971929117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971929118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,124 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,231 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/b101afc5162b47909ec7a340d11f4309 2024-11-18T23:17:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/55ac0c1b5bb847dbb9c169b3ffdd0e90 is 50, key is test_row_0/C:col10/1731971868378/Put/seqid=0 2024-11-18T23:17:49,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742400_1576 (size=12301) 2024-11-18T23:17:49,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/55ac0c1b5bb847dbb9c169b3ffdd0e90 2024-11-18T23:17:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/691cf79d72b54e899fba281e4a9f12ec as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/691cf79d72b54e899fba281e4a9f12ec 2024-11-18T23:17:49,293 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/691cf79d72b54e899fba281e4a9f12ec, entries=150, sequenceid=379, filesize=30.5 K 2024-11-18T23:17:49,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/b101afc5162b47909ec7a340d11f4309 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/b101afc5162b47909ec7a340d11f4309 2024-11-18T23:17:49,305 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/b101afc5162b47909ec7a340d11f4309, entries=150, sequenceid=379, filesize=12.0 K 2024-11-18T23:17:49,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/55ac0c1b5bb847dbb9c169b3ffdd0e90 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/55ac0c1b5bb847dbb9c169b3ffdd0e90 2024-11-18T23:17:49,313 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/55ac0c1b5bb847dbb9c169b3ffdd0e90, entries=150, sequenceid=379, filesize=12.0 K 2024-11-18T23:17:49,314 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~416.72 KB/426720, currentSize=40.25 KB/41220 for b28daae718a8c78c9acc9f3a87d1c727 in 702ms, sequenceid=379, compaction requested=true 2024-11-18T23:17:49,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:49,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:49,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-18T23:17:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-18T23:17:49,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-18T23:17:49,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9910 sec 2024-11-18T23:17:49,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.9980 sec 2024-11-18T23:17:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:49,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-18T23:17:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:49,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183d66e201d74b4b569dd2fe8cb8a44929_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971869430/Put/seqid=0 2024-11-18T23:17:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-18T23:17:49,470 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:49,471 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-18T23:17:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T23:17:49,485 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:49,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742401_1577 (size=14994) 2024-11-18T23:17:49,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:49,486 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:49,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971869478, max timestamp=1731971869497Num uniques 859; 2024-11-18T23:17:49,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:49,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971869478, max timestamp=1731971869497Num uniques 859; 2024-11-18T23:17:49,500 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:49,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971869478, max timestamp=1731971869497Num uniques 859; 2024-11-18T23:17:49,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:49,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971929547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971929547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971929552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971929552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T23:17:49,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971929587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T23:17:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:49,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971929657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971929667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971929668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971929668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T23:17:49,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T23:17:49,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:49,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:49,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:49,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971929869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,882 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:49,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971929877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971929887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971929888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:49,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:49,908 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411183d66e201d74b4b569dd2fe8cb8a44929_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183d66e201d74b4b569dd2fe8cb8a44929_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:49,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/0e476e9166e1410a9fc07e2025f3d7a3, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:49,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/0e476e9166e1410a9fc07e2025f3d7a3 is 175, key is test_row_0/A:col10/1731971869430/Put/seqid=0 2024-11-18T23:17:49,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742402_1578 (size=39949) 2024-11-18T23:17:49,955 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=389, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/0e476e9166e1410a9fc07e2025f3d7a3 2024-11-18T23:17:49,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T23:17:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:49,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:49,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:49,969 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:50,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/a8f886dfef904f72b8fca964778ef827 is 50, key is test_row_0/B:col10/1731971869430/Put/seqid=0 2024-11-18T23:17:50,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742403_1579 (size=12301) 2024-11-18T23:17:50,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/a8f886dfef904f72b8fca964778ef827 2024-11-18T23:17:50,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/c2b6484e2eb446d79c74d2e997915790 is 50, key is test_row_0/C:col10/1731971869430/Put/seqid=0 2024-11-18T23:17:50,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T23:17:50,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742404_1580 (size=12301) 2024-11-18T23:17:50,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T23:17:50,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:50,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:50,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:50,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:50,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:50,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/c2b6484e2eb446d79c74d2e997915790 2024-11-18T23:17:50,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:50,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/0e476e9166e1410a9fc07e2025f3d7a3 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/0e476e9166e1410a9fc07e2025f3d7a3 2024-11-18T23:17:50,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/0e476e9166e1410a9fc07e2025f3d7a3, entries=200, sequenceid=389, filesize=39.0 K 2024-11-18T23:17:50,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/a8f886dfef904f72b8fca964778ef827 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/a8f886dfef904f72b8fca964778ef827 2024-11-18T23:17:50,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/a8f886dfef904f72b8fca964778ef827, entries=150, sequenceid=389, filesize=12.0 K 2024-11-18T23:17:50,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/c2b6484e2eb446d79c74d2e997915790 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c2b6484e2eb446d79c74d2e997915790 2024-11-18T23:17:50,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971930190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971930191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971930197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971930197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c2b6484e2eb446d79c74d2e997915790, entries=150, sequenceid=389, filesize=12.0 K 2024-11-18T23:17:50,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for b28daae718a8c78c9acc9f3a87d1c727 in 797ms, sequenceid=389, compaction requested=true 2024-11-18T23:17:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:50,229 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:50,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:50,231 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:50,232 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134042 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:50,233 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/A is initiating minor compaction (all files) 2024-11-18T23:17:50,233 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/A in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:50,233 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/48bfd3946f9f4da59fb03f75c8890a11, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/782fb73a6f8d4bf88c43e8f1558f1759, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/691cf79d72b54e899fba281e4a9f12ec, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/0e476e9166e1410a9fc07e2025f3d7a3] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=130.9 K 2024-11-18T23:17:50,233 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:50,233 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/48bfd3946f9f4da59fb03f75c8890a11, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/782fb73a6f8d4bf88c43e8f1558f1759, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/691cf79d72b54e899fba281e4a9f12ec, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/0e476e9166e1410a9fc07e2025f3d7a3] 2024-11-18T23:17:50,234 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:50,234 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/B is initiating minor compaction (all files) 2024-11-18T23:17:50,234 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/B in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:50,234 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f43168a7acbd43caa6f504c2e138fde0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ba3842927d3f486181eb7ba972f5b170, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/b101afc5162b47909ec7a340d11f4309, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/a8f886dfef904f72b8fca964778ef827] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=48.4 K 2024-11-18T23:17:50,235 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48bfd3946f9f4da59fb03f75c8890a11, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971866413 2024-11-18T23:17:50,237 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f43168a7acbd43caa6f504c2e138fde0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971866413 2024-11-18T23:17:50,237 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782fb73a6f8d4bf88c43e8f1558f1759, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731971867125 2024-11-18T23:17:50,241 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3842927d3f486181eb7ba972f5b170, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731971867125 2024-11-18T23:17:50,241 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 691cf79d72b54e899fba281e4a9f12ec, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731971868378 2024-11-18T23:17:50,244 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e476e9166e1410a9fc07e2025f3d7a3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731971868748 2024-11-18T23:17:50,244 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting b101afc5162b47909ec7a340d11f4309, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731971868378 2024-11-18T23:17:50,246 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting a8f886dfef904f72b8fca964778ef827, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731971868748 2024-11-18T23:17:50,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-18T23:17:50,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:50,288 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-18T23:17:50,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:50,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:50,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:50,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:50,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:50,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:50,292 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:50,332 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#B#compaction#489 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:50,332 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/0541af3ca4d142749c15932ae558ced9 is 50, key is test_row_0/B:col10/1731971869430/Put/seqid=0 2024-11-18T23:17:50,337 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111866267e22f7014e5db44b4b6b2e004b72_b28daae718a8c78c9acc9f3a87d1c727 store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:50,340 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111866267e22f7014e5db44b4b6b2e004b72_b28daae718a8c78c9acc9f3a87d1c727, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:50,340 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111866267e22f7014e5db44b4b6b2e004b72_b28daae718a8c78c9acc9f3a87d1c727 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:50,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742405_1581 (size=12915) 2024-11-18T23:17:50,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118453c49039c594ec3a18c029453723493_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971869530/Put/seqid=0 2024-11-18T23:17:50,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742406_1582 (size=4469) 2024-11-18T23:17:50,404 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#A#compaction#488 average throughput is 0.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:50,404 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/1ac8124d72994a778c82f117c6f5a18c is 175, key is test_row_0/A:col10/1731971869430/Put/seqid=0 2024-11-18T23:17:50,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742407_1583 (size=12454) 2024-11-18T23:17:50,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742408_1584 (size=31869) 2024-11-18T23:17:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T23:17:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:50,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:50,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971930700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971930708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971930708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971930708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,710 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971930717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,769 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/0541af3ca4d142749c15932ae558ced9 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/0541af3ca4d142749c15932ae558ced9 2024-11-18T23:17:50,773 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/B of b28daae718a8c78c9acc9f3a87d1c727 into 0541af3ca4d142749c15932ae558ced9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:50,773 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:50,773 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/B, priority=12, startTime=1731971870229; duration=0sec 2024-11-18T23:17:50,774 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:50,774 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:50,774 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-18T23:17:50,780 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-18T23:17:50,780 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/C is initiating minor compaction (all files) 2024-11-18T23:17:50,780 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/C in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:50,780 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b59384bc28eb4ab481ee1d079e629646, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/02fc5841bbdc4de78969e0486865935d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/55ac0c1b5bb847dbb9c169b3ffdd0e90, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c2b6484e2eb446d79c74d2e997915790] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=48.4 K 2024-11-18T23:17:50,780 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting b59384bc28eb4ab481ee1d079e629646, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731971866413 2024-11-18T23:17:50,781 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 02fc5841bbdc4de78969e0486865935d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731971867125 2024-11-18T23:17:50,781 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 55ac0c1b5bb847dbb9c169b3ffdd0e90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731971868378 2024-11-18T23:17:50,783 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b6484e2eb446d79c74d2e997915790, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731971868748 2024-11-18T23:17:50,810 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#C#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:50,811 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/295e30e495df4de580f450e0a002fcf2 is 50, key is test_row_0/C:col10/1731971869430/Put/seqid=0 2024-11-18T23:17:50,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:50,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971930817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:50,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:50,823 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118453c49039c594ec3a18c029453723493_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118453c49039c594ec3a18c029453723493_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:50,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ecb3ea4bd436427d85cc644e3e270a0b, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ecb3ea4bd436427d85cc644e3e270a0b is 175, key is test_row_0/A:col10/1731971869530/Put/seqid=0 2024-11-18T23:17:50,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742410_1586 (size=31255) 2024-11-18T23:17:50,850 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/1ac8124d72994a778c82f117c6f5a18c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1ac8124d72994a778c82f117c6f5a18c 2024-11-18T23:17:50,859 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/A of b28daae718a8c78c9acc9f3a87d1c727 into 1ac8124d72994a778c82f117c6f5a18c(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:50,859 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:50,859 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/A, priority=12, startTime=1731971870229; duration=0sec 2024-11-18T23:17:50,859 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:50,859 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:50,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742409_1585 (size=12915) 2024-11-18T23:17:50,886 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/295e30e495df4de580f450e0a002fcf2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/295e30e495df4de580f450e0a002fcf2 2024-11-18T23:17:50,893 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/C of b28daae718a8c78c9acc9f3a87d1c727 into 295e30e495df4de580f450e0a002fcf2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:50,893 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:50,893 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/C, priority=12, startTime=1731971870229; duration=0sec 2024-11-18T23:17:50,893 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:50,893 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971931029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:51,247 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ecb3ea4bd436427d85cc644e3e270a0b 2024-11-18T23:17:51,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/bdd70a70dd1c49c083e5a5c8d54ed077 is 50, key is test_row_0/B:col10/1731971869530/Put/seqid=0 2024-11-18T23:17:51,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742411_1587 (size=12301) 2024-11-18T23:17:51,286 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/bdd70a70dd1c49c083e5a5c8d54ed077 2024-11-18T23:17:51,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/ef94bcc6b6a84980a86abe61e67f5282 is 50, key is test_row_0/C:col10/1731971869530/Put/seqid=0 2024-11-18T23:17:51,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742412_1588 (size=12301) 2024-11-18T23:17:51,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971931347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T23:17:51,709 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/ef94bcc6b6a84980a86abe61e67f5282 2024-11-18T23:17:51,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ecb3ea4bd436427d85cc644e3e270a0b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ecb3ea4bd436427d85cc644e3e270a0b 2024-11-18T23:17:51,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971931718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:51,726 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971931718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:51,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971931728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:51,736 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:51,739 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ecb3ea4bd436427d85cc644e3e270a0b, entries=150, sequenceid=426, filesize=30.5 K 2024-11-18T23:17:51,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/bdd70a70dd1c49c083e5a5c8d54ed077 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/bdd70a70dd1c49c083e5a5c8d54ed077 2024-11-18T23:17:51,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971931736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:51,743 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:51,744 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:51,745 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:51,750 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/bdd70a70dd1c49c083e5a5c8d54ed077, entries=150, sequenceid=426, filesize=12.0 K 2024-11-18T23:17:51,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/ef94bcc6b6a84980a86abe61e67f5282 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ef94bcc6b6a84980a86abe61e67f5282 2024-11-18T23:17:51,775 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ef94bcc6b6a84980a86abe61e67f5282, entries=150, sequenceid=426, filesize=12.0 K 2024-11-18T23:17:51,777 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for b28daae718a8c78c9acc9f3a87d1c727 in 1489ms, sequenceid=426, compaction requested=false 2024-11-18T23:17:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:51,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-18T23:17:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-18T23:17:51,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-18T23:17:51,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2950 sec 2024-11-18T23:17:51,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.3140 sec 2024-11-18T23:17:51,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:17:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:51,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:51,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180aba8bf3440645119d27f22469a24dba_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971871860/Put/seqid=0 2024-11-18T23:17:51,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742413_1589 (size=12454) 2024-11-18T23:17:52,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971871981, max timestamp=1731971872036Num uniques 859; 2024-11-18T23:17:52,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:52,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971871981, max timestamp=1731971872036Num uniques 859; 2024-11-18T23:17:52,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:17:52,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731971871981, max timestamp=1731971872036Num uniques 859; 2024-11-18T23:17:52,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-18T23:17:52,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971932221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:52,312 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:52,332 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411180aba8bf3440645119d27f22469a24dba_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180aba8bf3440645119d27f22469a24dba_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:52,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/85ede9b42a8a4d6ca1519a7e4b40eb75, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:52,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/85ede9b42a8a4d6ca1519a7e4b40eb75 is 175, key is test_row_0/A:col10/1731971871860/Put/seqid=0 2024-11-18T23:17:52,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971932352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:52,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742414_1590 (size=31255) 2024-11-18T23:17:52,384 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/85ede9b42a8a4d6ca1519a7e4b40eb75 2024-11-18T23:17:52,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/1fb0a43981d7454ca2f26464081e85f0 is 50, key is test_row_0/B:col10/1731971871860/Put/seqid=0 2024-11-18T23:17:52,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742415_1591 (size=12301) 2024-11-18T23:17:52,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971932568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:52,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/1fb0a43981d7454ca2f26464081e85f0 2024-11-18T23:17:52,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:52,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971932877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:52,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:52,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/e5dcb60916f1484c9033b96a2d147659 is 50, key is test_row_0/C:col10/1731971871860/Put/seqid=0 2024-11-18T23:17:52,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742416_1592 (size=12301) 2024-11-18T23:17:52,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/e5dcb60916f1484c9033b96a2d147659 2024-11-18T23:17:52,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/85ede9b42a8a4d6ca1519a7e4b40eb75 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/85ede9b42a8a4d6ca1519a7e4b40eb75 2024-11-18T23:17:52,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/85ede9b42a8a4d6ca1519a7e4b40eb75, entries=150, sequenceid=440, filesize=30.5 K 2024-11-18T23:17:52,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/1fb0a43981d7454ca2f26464081e85f0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/1fb0a43981d7454ca2f26464081e85f0 2024-11-18T23:17:52,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/1fb0a43981d7454ca2f26464081e85f0, entries=150, sequenceid=440, filesize=12.0 K 2024-11-18T23:17:52,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/e5dcb60916f1484c9033b96a2d147659 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/e5dcb60916f1484c9033b96a2d147659 2024-11-18T23:17:52,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/e5dcb60916f1484c9033b96a2d147659, entries=150, sequenceid=440, filesize=12.0 K 2024-11-18T23:17:52,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.68 KB/129720 for b28daae718a8c78c9acc9f3a87d1c727 in 1062ms, sequenceid=440, compaction requested=true 2024-11-18T23:17:52,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:52,928 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:52,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-18T23:17:52,930 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:52,931 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:52,932 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/A is initiating minor compaction (all files) 2024-11-18T23:17:52,932 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/A in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:52,932 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1ac8124d72994a778c82f117c6f5a18c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ecb3ea4bd436427d85cc644e3e270a0b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/85ede9b42a8a4d6ca1519a7e4b40eb75] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=92.2 K 2024-11-18T23:17:52,932 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:52,932 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1ac8124d72994a778c82f117c6f5a18c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ecb3ea4bd436427d85cc644e3e270a0b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/85ede9b42a8a4d6ca1519a7e4b40eb75] 2024-11-18T23:17:52,932 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ac8124d72994a778c82f117c6f5a18c, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731971868748 2024-11-18T23:17:52,932 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:52,933 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/B is initiating minor compaction (all files) 2024-11-18T23:17:52,933 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/B in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:52,933 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/0541af3ca4d142749c15932ae558ced9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/bdd70a70dd1c49c083e5a5c8d54ed077, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/1fb0a43981d7454ca2f26464081e85f0] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=36.6 K 2024-11-18T23:17:52,933 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ecb3ea4bd436427d85cc644e3e270a0b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731971869530 2024-11-18T23:17:52,933 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 0541af3ca4d142749c15932ae558ced9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731971868748 2024-11-18T23:17:52,933 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85ede9b42a8a4d6ca1519a7e4b40eb75, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731971870662 2024-11-18T23:17:52,933 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting bdd70a70dd1c49c083e5a5c8d54ed077, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731971869530 2024-11-18T23:17:52,934 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb0a43981d7454ca2f26464081e85f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731971870662 2024-11-18T23:17:52,945 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#B#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:52,945 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/ef79277681c34b9da1d5c1db122be126 is 50, key is test_row_0/B:col10/1731971871860/Put/seqid=0 2024-11-18T23:17:52,956 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:52,966 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411181cdebae0ffe3471e91dcc9ff16ae283f_b28daae718a8c78c9acc9f3a87d1c727 store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:52,971 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411181cdebae0ffe3471e91dcc9ff16ae283f_b28daae718a8c78c9acc9f3a87d1c727, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:52,971 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411181cdebae0ffe3471e91dcc9ff16ae283f_b28daae718a8c78c9acc9f3a87d1c727 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:52,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742417_1593 (size=13017) 2024-11-18T23:17:52,990 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/ef79277681c34b9da1d5c1db122be126 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ef79277681c34b9da1d5c1db122be126 2024-11-18T23:17:52,997 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/B of b28daae718a8c78c9acc9f3a87d1c727 into ef79277681c34b9da1d5c1db122be126(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:52,997 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:52,997 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/B, priority=13, startTime=1731971872928; duration=0sec 2024-11-18T23:17:52,998 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:52,998 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:52,998 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:52,999 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:52,999 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/C is initiating minor compaction (all files) 2024-11-18T23:17:52,999 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/C in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:52,999 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/295e30e495df4de580f450e0a002fcf2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ef94bcc6b6a84980a86abe61e67f5282, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/e5dcb60916f1484c9033b96a2d147659] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=36.6 K 2024-11-18T23:17:52,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742418_1594 (size=4469) 2024-11-18T23:17:53,000 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 295e30e495df4de580f450e0a002fcf2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731971868748 2024-11-18T23:17:53,001 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ef94bcc6b6a84980a86abe61e67f5282, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731971869530 2024-11-18T23:17:53,001 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting e5dcb60916f1484c9033b96a2d147659, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731971870662 2024-11-18T23:17:53,003 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#A#compaction#498 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:53,004 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ea26061fb2dc4a17a27ced121edb5796 is 175, key is test_row_0/A:col10/1731971871860/Put/seqid=0 2024-11-18T23:17:53,030 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#C#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:53,030 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/f709d57586b6452685eb4789c63128be is 50, key is test_row_0/C:col10/1731971871860/Put/seqid=0 2024-11-18T23:17:53,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742419_1595 (size=31971) 2024-11-18T23:17:53,062 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ea26061fb2dc4a17a27ced121edb5796 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ea26061fb2dc4a17a27ced121edb5796 2024-11-18T23:17:53,069 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/A of b28daae718a8c78c9acc9f3a87d1c727 into ea26061fb2dc4a17a27ced121edb5796(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:53,069 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:53,069 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/A, priority=13, startTime=1731971872927; duration=0sec 2024-11-18T23:17:53,069 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:53,069 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:53,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742420_1596 (size=13017) 2024-11-18T23:17:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:53,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=133.39 KB heapSize=383.16 KB 2024-11-18T23:17:53,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:53,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:53,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:53,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:53,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:53,405 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:53,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188fed30c8abee48dabfbf4ee9356ffce8_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971873403/Put/seqid=0 2024-11-18T23:17:53,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742421_1597 (size=14994) 2024-11-18T23:17:53,463 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:53,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971933459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:53,468 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411188fed30c8abee48dabfbf4ee9356ffce8_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fed30c8abee48dabfbf4ee9356ffce8_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:53,470 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ae240cee298e427085a08d25af9f1e03, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:53,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ae240cee298e427085a08d25af9f1e03 is 175, key is test_row_0/A:col10/1731971873403/Put/seqid=0 2024-11-18T23:17:53,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742422_1598 (size=39949) 2024-11-18T23:17:53,480 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=479, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ae240cee298e427085a08d25af9f1e03 2024-11-18T23:17:53,486 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/f709d57586b6452685eb4789c63128be as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f709d57586b6452685eb4789c63128be 2024-11-18T23:17:53,494 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/C of b28daae718a8c78c9acc9f3a87d1c727 into f709d57586b6452685eb4789c63128be(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:53,494 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:53,494 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/C, priority=13, startTime=1731971872928; duration=0sec 2024-11-18T23:17:53,494 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:53,494 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:53,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f9fa07e8877744e9ae6ce1129f63d997 is 50, key is test_row_0/B:col10/1731971873403/Put/seqid=0 2024-11-18T23:17:53,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742423_1599 (size=12301) 2024-11-18T23:17:53,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:53,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971933577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:53,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-18T23:17:53,638 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:53,639 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-18T23:17:53,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-18T23:17:53,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T23:17:53,642 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-18T23:17:53,643 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-18T23:17:53,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-18T23:17:53,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:53,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32926 deadline: 1731971933737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:53,740 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,740 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,741 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T23:17:53,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32958 deadline: 1731971933758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:53,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32968 deadline: 1731971933758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:53,762 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_1' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:53,763 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:53,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:53,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32942 deadline: 1731971933767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:53,770 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9d96596e55a9,32899,1731971713394 for 'test_row_2' in b28daae718a8c78c9acc9f3a87d1c727 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-18T23:17:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,771 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:53,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971933787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:53,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:53,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:53,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:53,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:53,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:53,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:53,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f9fa07e8877744e9ae6ce1129f63d997 2024-11-18T23:17:53,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/0623809361964462b6ae4c1d8c9755e2 is 50, key is test_row_0/C:col10/1731971873403/Put/seqid=0 2024-11-18T23:17:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T23:17:53,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:53,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:53,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:53,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742424_1600 (size=12301) 2024-11-18T23:17:53,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/0623809361964462b6ae4c1d8c9755e2 2024-11-18T23:17:53,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/ae240cee298e427085a08d25af9f1e03 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae240cee298e427085a08d25af9f1e03 2024-11-18T23:17:53,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae240cee298e427085a08d25af9f1e03, entries=200, sequenceid=479, filesize=39.0 K 2024-11-18T23:17:53,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/f9fa07e8877744e9ae6ce1129f63d997 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f9fa07e8877744e9ae6ce1129f63d997 2024-11-18T23:17:54,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f9fa07e8877744e9ae6ce1129f63d997, entries=150, sequenceid=479, filesize=12.0 K 2024-11-18T23:17:54,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/0623809361964462b6ae4c1d8c9755e2 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/0623809361964462b6ae4c1d8c9755e2 2024-11-18T23:17:54,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/0623809361964462b6ae4c1d8c9755e2, entries=150, sequenceid=479, filesize=12.0 K 2024-11-18T23:17:54,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.39 KB/136590, heapSize ~383.11 KB/392304, currentSize=46.96 KB/48090 for b28daae718a8c78c9acc9f3a87d1c727 in 602ms, sequenceid=479, compaction requested=false 2024-11-18T23:17:54,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:54,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-18T23:17:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:54,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:54,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:54,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:54,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:54,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:54,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:54,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:54,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:54,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118613b6ea22fd74f0eb9914f793879b545_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971873451/Put/seqid=0 2024-11-18T23:17:54,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742425_1601 (size=14994) 2024-11-18T23:17:54,184 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:54,190 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118613b6ea22fd74f0eb9914f793879b545_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118613b6ea22fd74f0eb9914f793879b545_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:54,191 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/238a856d41d848ff9faaf7f42158df07, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:54,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/238a856d41d848ff9faaf7f42158df07 is 175, key is test_row_0/A:col10/1731971873451/Put/seqid=0 2024-11-18T23:17:54,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742426_1602 (size=39949) 2024-11-18T23:17:54,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=491, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/238a856d41d848ff9faaf7f42158df07 2024-11-18T23:17:54,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/56c0d1a36a2f47e0907550f30fa63b2c is 50, key is test_row_0/B:col10/1731971873451/Put/seqid=0 2024-11-18T23:17:54,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T23:17:54,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742427_1603 (size=12301) 2024-11-18T23:17:54,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/56c0d1a36a2f47e0907550f30fa63b2c 2024-11-18T23:17:54,292 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:54,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:54,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,292 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971874233, max timestamp=1731971874304Num uniques 859; 2024-11-18T23:17:54,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971874233, max timestamp=1731971874304Num uniques 859; 2024-11-18T23:17:54,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731971874233, max timestamp=1731971874304Num uniques 859; 2024-11-18T23:17:54,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:54,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:54,307 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9d96596e55a9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-18T23:17:54,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/721f1f74303b44109fb16c6da47b4459 is 50, key is test_row_0/C:col10/1731971873451/Put/seqid=0 2024-11-18T23:17:54,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742428_1604 (size=12301) 2024-11-18T23:17:54,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=491 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/721f1f74303b44109fb16c6da47b4459 2024-11-18T23:17:54,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/238a856d41d848ff9faaf7f42158df07 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/238a856d41d848ff9faaf7f42158df07 2024-11-18T23:17:54,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/238a856d41d848ff9faaf7f42158df07, entries=200, sequenceid=491, filesize=39.0 K 2024-11-18T23:17:54,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/56c0d1a36a2f47e0907550f30fa63b2c as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/56c0d1a36a2f47e0907550f30fa63b2c 2024-11-18T23:17:54,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/56c0d1a36a2f47e0907550f30fa63b2c, entries=150, sequenceid=491, filesize=12.0 K 2024-11-18T23:17:54,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/721f1f74303b44109fb16c6da47b4459 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/721f1f74303b44109fb16c6da47b4459 2024-11-18T23:17:54,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/721f1f74303b44109fb16c6da47b4459, entries=150, sequenceid=491, filesize=12.0 K 2024-11-18T23:17:54,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=67.09 KB/68700 for b28daae718a8c78c9acc9f3a87d1c727 in 262ms, sequenceid=491, compaction requested=true 2024-11-18T23:17:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:54,381 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:54,381 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 3 2024-11-18T23:17:54,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:54,384 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111869 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:54,384 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/A is initiating minor compaction (all files) 2024-11-18T23:17:54,384 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/A in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,384 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:54,384 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ea26061fb2dc4a17a27ced121edb5796, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae240cee298e427085a08d25af9f1e03, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/238a856d41d848ff9faaf7f42158df07] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=109.2 K 2024-11-18T23:17:54,384 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/B is initiating minor compaction (all files) 2024-11-18T23:17:54,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/B in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,384 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,384 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. files: [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ea26061fb2dc4a17a27ced121edb5796, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae240cee298e427085a08d25af9f1e03, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/238a856d41d848ff9faaf7f42158df07] 2024-11-18T23:17:54,384 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ef79277681c34b9da1d5c1db122be126, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f9fa07e8877744e9ae6ce1129f63d997, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/56c0d1a36a2f47e0907550f30fa63b2c] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=36.7 K 2024-11-18T23:17:54,386 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting ef79277681c34b9da1d5c1db122be126, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731971870662 2024-11-18T23:17:54,386 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea26061fb2dc4a17a27ced121edb5796, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731971870662 2024-11-18T23:17:54,386 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f9fa07e8877744e9ae6ce1129f63d997, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731971872162 2024-11-18T23:17:54,387 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae240cee298e427085a08d25af9f1e03, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731971872162 2024-11-18T23:17:54,387 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] compactions.Compactor(225): Compacting 238a856d41d848ff9faaf7f42158df07, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731971873427 2024-11-18T23:17:54,387 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 56c0d1a36a2f47e0907550f30fa63b2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731971873427 2024-11-18T23:17:54,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:54,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=73.80 KB heapSize=205.69 KB 2024-11-18T23:17:54,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:54,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:54,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:54,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:54,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:54,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-18T23:17:54,410 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#B#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:54,411 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/13d44a22b5bf469fb4e1af2a713ce2f8 is 50, key is test_row_0/B:col10/1731971873451/Put/seqid=0 2024-11-18T23:17:54,412 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:54,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185bdb18cd3f224a7088326e802867b273_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971874304/Put/seqid=0 2024-11-18T23:17:54,418 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111805d4da00af55482197e4cb3c202a2c06_b28daae718a8c78c9acc9f3a87d1c727 store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:54,420 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111805d4da00af55482197e4cb3c202a2c06_b28daae718a8c78c9acc9f3a87d1c727, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:54,420 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111805d4da00af55482197e4cb3c202a2c06_b28daae718a8c78c9acc9f3a87d1c727 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:54,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742429_1605 (size=13119) 2024-11-18T23:17:54,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742431_1607 (size=4469) 2024-11-18T23:17:54,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742430_1606 (size=14994) 2024-11-18T23:17:54,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:54,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971934532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:54,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:54,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971934648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:54,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T23:17:54,856 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#A#compaction#507 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:54,857 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/f4d0f8fa0b314de1b8a49bf3b8f9a195 is 175, key is test_row_0/A:col10/1731971873451/Put/seqid=0 2024-11-18T23:17:54,859 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/13d44a22b5bf469fb4e1af2a713ce2f8 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/13d44a22b5bf469fb4e1af2a713ce2f8 2024-11-18T23:17:54,863 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/B of b28daae718a8c78c9acc9f3a87d1c727 into 13d44a22b5bf469fb4e1af2a713ce2f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:54,863 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:54,863 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/B, priority=13, startTime=1731971874381; duration=0sec 2024-11-18T23:17:54,864 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-18T23:17:54,864 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:54,864 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-18T23:17:54,864 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-18T23:17:54,864 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1541): b28daae718a8c78c9acc9f3a87d1c727/C is initiating minor compaction (all files) 2024-11-18T23:17:54,864 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b28daae718a8c78c9acc9f3a87d1c727/C in TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,865 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f709d57586b6452685eb4789c63128be, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/0623809361964462b6ae4c1d8c9755e2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/721f1f74303b44109fb16c6da47b4459] into tmpdir=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp, totalSize=36.7 K 2024-11-18T23:17:54,865 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting f709d57586b6452685eb4789c63128be, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731971870662 2024-11-18T23:17:54,865 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 0623809361964462b6ae4c1d8c9755e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731971872162 2024-11-18T23:17:54,866 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] compactions.Compactor(225): Compacting 721f1f74303b44109fb16c6da47b4459, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=491, earliestPutTs=1731971873427 2024-11-18T23:17:54,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971934872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:54,873 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b28daae718a8c78c9acc9f3a87d1c727#C#compaction#509 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-18T23:17:54,874 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/38227a72685d47b388990587c0586f46 is 50, key is test_row_0/C:col10/1731971873451/Put/seqid=0 2024-11-18T23:17:54,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742432_1608 (size=32073) 2024-11-18T23:17:54,890 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:54,893 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411185bdb18cd3f224a7088326e802867b273_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185bdb18cd3f224a7088326e802867b273_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:54,893 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/a2c4ba39bb7446989d220cc6fc1675a4, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:54,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/a2c4ba39bb7446989d220cc6fc1675a4 is 175, key is test_row_0/A:col10/1731971874304/Put/seqid=0 2024-11-18T23:17:54,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742433_1609 (size=13119) 2024-11-18T23:17:54,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:54,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:54,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:54,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:54,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742434_1610 (size=39949) 2024-11-18T23:17:54,943 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=519, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/a2c4ba39bb7446989d220cc6fc1675a4 2024-11-18T23:17:54,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/d59b98f4d56c4bcbb0db0abe752c2a85 is 50, key is test_row_0/B:col10/1731971874304/Put/seqid=0 2024-11-18T23:17:55,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742435_1611 (size=12301) 2024-11-18T23:17:55,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/d59b98f4d56c4bcbb0db0abe752c2a85 2024-11-18T23:17:55,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/b3ae505cb2864efaa3ec83b62f1ad513 is 50, key is test_row_0/C:col10/1731971874304/Put/seqid=0 2024-11-18T23:17:55,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742436_1612 (size=12301) 2024-11-18T23:17:55,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:55,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971935177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:55,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:55,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:55,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,306 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/f4d0f8fa0b314de1b8a49bf3b8f9a195 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f4d0f8fa0b314de1b8a49bf3b8f9a195 2024-11-18T23:17:55,329 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/38227a72685d47b388990587c0586f46 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/38227a72685d47b388990587c0586f46 2024-11-18T23:17:55,341 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/A of b28daae718a8c78c9acc9f3a87d1c727 into f4d0f8fa0b314de1b8a49bf3b8f9a195(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:55,341 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:55,341 INFO [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/A, priority=13, startTime=1731971874381; duration=0sec 2024-11-18T23:17:55,341 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:55,341 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:55,345 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b28daae718a8c78c9acc9f3a87d1c727/C of b28daae718a8c78c9acc9f3a87d1c727 into 38227a72685d47b388990587c0586f46(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-18T23:17:55,345 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:55,345 INFO [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., storeName=b28daae718a8c78c9acc9f3a87d1c727/C, priority=13, startTime=1731971874381; duration=0sec 2024-11-18T23:17:55,345 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:55,345 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:55,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:55,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:55,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:55,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:55,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T23:17:55,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/b3ae505cb2864efaa3ec83b62f1ad513 2024-11-18T23:17:55,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/a2c4ba39bb7446989d220cc6fc1675a4 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/a2c4ba39bb7446989d220cc6fc1675a4 2024-11-18T23:17:55,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/a2c4ba39bb7446989d220cc6fc1675a4, entries=200, sequenceid=519, filesize=39.0 K 2024-11-18T23:17:55,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/d59b98f4d56c4bcbb0db0abe752c2a85 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/d59b98f4d56c4bcbb0db0abe752c2a85 2024-11-18T23:17:55,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/d59b98f4d56c4bcbb0db0abe752c2a85, entries=150, sequenceid=519, filesize=12.0 K 2024-11-18T23:17:55,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/b3ae505cb2864efaa3ec83b62f1ad513 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b3ae505cb2864efaa3ec83b62f1ad513 2024-11-18T23:17:55,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b3ae505cb2864efaa3ec83b62f1ad513, entries=150, sequenceid=519, filesize=12.0 K 2024-11-18T23:17:55,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.64 KB/210576, currentSize=107.34 KB/109920 for b28daae718a8c78c9acc9f3a87d1c727 in 1136ms, sequenceid=519, compaction requested=false 2024-11-18T23:17:55,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:55,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32899 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-18T23:17:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:55,559 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-18T23:17:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:55,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b337d4bbb46a478bb3961475734bd7de_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971874525/Put/seqid=0 2024-11-18T23:17:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742437_1613 (size=12454) 2024-11-18T23:17:55,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:55,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. as already flushing 2024-11-18T23:17:55,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:55,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971935736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:55,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T23:17:55,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:55,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971935848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:55,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:55,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:55,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:55,971 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118b337d4bbb46a478bb3961475734bd7de_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b337d4bbb46a478bb3961475734bd7de_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:55,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/5ac4cc0b1aaa47a39d81538378e7d28b, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:55,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/5ac4cc0b1aaa47a39d81538378e7d28b is 175, key is test_row_0/A:col10/1731971874525/Put/seqid=0 2024-11-18T23:17:55,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742438_1614 (size=31255) 2024-11-18T23:17:55,991 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=541, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/5ac4cc0b1aaa47a39d81538378e7d28b 2024-11-18T23:17:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/3230e4cd7daa4f58950540d9b8545917 is 50, key is test_row_0/B:col10/1731971874525/Put/seqid=0 2024-11-18T23:17:56,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742439_1615 (size=12301) 2024-11-18T23:17:56,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971936058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:56,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-18T23:17:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32899 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32922 deadline: 1731971936367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 , the old value is region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-18T23:17:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., hostname=9d96596e55a9,32899,1731971713394, seqNum=5 because the exception is null or not the one we care about 2024-11-18T23:17:56,426 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/3230e4cd7daa4f58950540d9b8545917 2024-11-18T23:17:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/cbca8db655da4d09b91ecac8c148ab66 is 50, key is test_row_0/C:col10/1731971874525/Put/seqid=0 2024-11-18T23:17:56,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742440_1616 (size=12301) 2024-11-18T23:17:56,455 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=541 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/cbca8db655da4d09b91ecac8c148ab66 2024-11-18T23:17:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/5ac4cc0b1aaa47a39d81538378e7d28b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/5ac4cc0b1aaa47a39d81538378e7d28b 2024-11-18T23:17:56,478 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/5ac4cc0b1aaa47a39d81538378e7d28b, entries=150, sequenceid=541, filesize=30.5 K 2024-11-18T23:17:56,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/3230e4cd7daa4f58950540d9b8545917 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3230e4cd7daa4f58950540d9b8545917 2024-11-18T23:17:56,484 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3230e4cd7daa4f58950540d9b8545917, entries=150, sequenceid=541, filesize=12.0 K 2024-11-18T23:17:56,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/cbca8db655da4d09b91ecac8c148ab66 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/cbca8db655da4d09b91ecac8c148ab66 2024-11-18T23:17:56,498 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/cbca8db655da4d09b91ecac8c148ab66, entries=150, sequenceid=541, filesize=12.0 K 2024-11-18T23:17:56,499 INFO [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for b28daae718a8c78c9acc9f3a87d1c727 in 940ms, sequenceid=541, compaction requested=true 2024-11-18T23:17:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9d96596e55a9:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-18T23:17:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-18T23:17:56,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-18T23:17:56,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8730 sec 2024-11-18T23:17:56,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.9090 sec 2024-11-18T23:17:56,649 INFO [Thread-2294 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2294. 2024-11-18T23:17:56,649 DEBUG [Thread-2294 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:56,649 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,649 DEBUG [Thread-2294 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:56,651 INFO [Thread-2296 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2296. 2024-11-18T23:17:56,651 DEBUG [Thread-2296 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:56,651 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,651 DEBUG [Thread-2296 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:56,652 INFO [Thread-2290 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2290. 2024-11-18T23:17:56,652 DEBUG [Thread-2290 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:56,652 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,652 DEBUG [Thread-2290 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,652 INFO [Thread-2298 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2298. 2024-11-18T23:17:56,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:56,652 DEBUG [Thread-2298 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:56,652 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:56,652 DEBUG [Thread-2298 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,654 INFO [Thread-2292 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2292. 2024-11-18T23:17:56,654 DEBUG [Thread-2292 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:56,654 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,654 DEBUG [Thread-2292 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:56,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32899 {}] regionserver.HRegion(8855): Flush requested on b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:56,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-18T23:17:56,881 INFO [Thread-2279 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2279. 2024-11-18T23:17:56,881 DEBUG [Thread-2279 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:56,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:56,881 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:56,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:56,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:56,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:56,882 DEBUG [Thread-2279 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:56,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:56,882 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:56,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118479d1afe9bf84d778a4e46cc913b3ef4_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_0/A:col10/1731971875706/Put/seqid=0 2024-11-18T23:17:56,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742441_1617 (size=12454) 2024-11-18T23:17:57,293 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:57,295 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118479d1afe9bf84d778a4e46cc913b3ef4_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118479d1afe9bf84d778a4e46cc913b3ef4_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:57,296 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/c642fcd76615494d8ef5d47b64b53def, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:57,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/c642fcd76615494d8ef5d47b64b53def is 175, key is test_row_0/A:col10/1731971875706/Put/seqid=0 2024-11-18T23:17:57,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742442_1618 (size=31255) 2024-11-18T23:17:57,709 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=555, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/c642fcd76615494d8ef5d47b64b53def 2024-11-18T23:17:57,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/05a42a41d66c4779ba5d692cec634dab is 50, key is test_row_0/B:col10/1731971875706/Put/seqid=0 2024-11-18T23:17:57,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742443_1619 (size=12301) 2024-11-18T23:17:57,768 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-18T23:17:57,769 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:57,769 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,769 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:57,789 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-18T23:17:57,789 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:57,789 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,789 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:57,798 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-18T23:17:57,798 INFO [Thread-2281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2281. 2024-11-18T23:17:57,798 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:57,798 DEBUG [Thread-2281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-18T23:17:57,798 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,798 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,798 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,798 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:57,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-18T23:17:57,807 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:17:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-18T23:17:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 183 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 83 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1608 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1584 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1550 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1562 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1565 2024-11-18T23:17:57,808 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-18T23:17:57,808 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:17:57,808 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:17:57,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:17:57,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:17:57,809 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-18T23:17:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-18T23:17:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T23:17:57,811 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971877811"}]},"ts":"1731971877811"} 2024-11-18T23:17:57,812 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-18T23:17:57,812 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-18T23:17:57,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-18T23:17:57,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, UNASSIGN}] 2024-11-18T23:17:57,814 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, UNASSIGN 2024-11-18T23:17:57,815 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=CLOSING, regionLocation=9d96596e55a9,32899,1731971713394 2024-11-18T23:17:57,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, UNASSIGN because future has completed 2024-11-18T23:17:57,816 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-18T23:17:57,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394}] 2024-11-18T23:17:57,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T23:17:57,969 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(122): Close b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:57,969 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-18T23:17:57,969 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1722): Closing b28daae718a8c78c9acc9f3a87d1c727, disabling compactions & flushes 2024-11-18T23:17:57,969 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:58,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T23:17:58,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/05a42a41d66c4779ba5d692cec634dab 2024-11-18T23:17:58,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/eee0db6f381f4b868dd0de17197e4a69 is 50, key is test_row_0/C:col10/1731971875706/Put/seqid=0 2024-11-18T23:17:58,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742444_1620 (size=12301) 2024-11-18T23:17:58,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T23:17:58,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/eee0db6f381f4b868dd0de17197e4a69 2024-11-18T23:17:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/c642fcd76615494d8ef5d47b64b53def as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/c642fcd76615494d8ef5d47b64b53def 2024-11-18T23:17:58,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/c642fcd76615494d8ef5d47b64b53def, entries=150, sequenceid=555, filesize=30.5 K 2024-11-18T23:17:58,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/05a42a41d66c4779ba5d692cec634dab as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/05a42a41d66c4779ba5d692cec634dab 2024-11-18T23:17:58,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/05a42a41d66c4779ba5d692cec634dab, entries=150, sequenceid=555, filesize=12.0 K 2024-11-18T23:17:58,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/eee0db6f381f4b868dd0de17197e4a69 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/eee0db6f381f4b868dd0de17197e4a69 2024-11-18T23:17:58,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/eee0db6f381f4b868dd0de17197e4a69, entries=150, sequenceid=555, filesize=12.0 K 2024-11-18T23:17:58,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for b28daae718a8c78c9acc9f3a87d1c727 in 1686ms, sequenceid=555, compaction requested=true 2024-11-18T23:17:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b28daae718a8c78c9acc9f3a87d1c727: 2024-11-18T23:17:58,567 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. because compaction request was cancelled 2024-11-18T23:17:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:A, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:58,567 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:A 2024-11-18T23:17:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:B, priority=-2147483648, current under compaction store size is 1 2024-11-18T23:17:58,567 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. because compaction request was cancelled 2024-11-18T23:17:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:58,567 DEBUG [RS:0;9d96596e55a9:32899-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:B 2024-11-18T23:17:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b28daae718a8c78c9acc9f3a87d1c727:C, priority=-2147483648, current under compaction store size is 2 2024-11-18T23:17:58,567 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. because compaction request was cancelled 2024-11-18T23:17:58,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-18T23:17:58,567 DEBUG [RS:0;9d96596e55a9:32899-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b28daae718a8c78c9acc9f3a87d1c727:C 2024-11-18T23:17:58,568 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:58,568 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:58,568 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. after waiting 0 ms 2024-11-18T23:17:58,568 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:17:58,568 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(2902): Flushing b28daae718a8c78c9acc9f3a87d1c727 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-18T23:17:58,569 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=A 2024-11-18T23:17:58,569 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:58,569 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=B 2024-11-18T23:17:58,569 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:58,569 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK b28daae718a8c78c9acc9f3a87d1c727, store=C 2024-11-18T23:17:58,569 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-18T23:17:58,576 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118575feb09b8e7429cb286c5b794d48aee_b28daae718a8c78c9acc9f3a87d1c727 is 50, key is test_row_1/A:col10/1731971877797/Put/seqid=0 2024-11-18T23:17:58,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742445_1621 (size=9914) 2024-11-18T23:17:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T23:17:58,979 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T23:17:58,983 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241118575feb09b8e7429cb286c5b794d48aee_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118575feb09b8e7429cb286c5b794d48aee_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:17:58,985 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/1dfd5aad8d0b46cb853a3c9ad84d0c44, store: [table=TestAcidGuarantees family=A region=b28daae718a8c78c9acc9f3a87d1c727] 2024-11-18T23:17:58,986 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/1dfd5aad8d0b46cb853a3c9ad84d0c44 is 175, key is test_row_1/A:col10/1731971877797/Put/seqid=0 2024-11-18T23:17:58,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742446_1622 (size=22561) 2024-11-18T23:17:59,395 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=562, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/1dfd5aad8d0b46cb853a3c9ad84d0c44 2024-11-18T23:17:59,401 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/da6b8a65d6ea4a8fb6eb25161106e9ed is 50, key is test_row_1/B:col10/1731971877797/Put/seqid=0 2024-11-18T23:17:59,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742447_1623 (size=9857) 2024-11-18T23:17:59,827 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/da6b8a65d6ea4a8fb6eb25161106e9ed 2024-11-18T23:17:59,832 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/69ae9cc6ef754d11b9109859d2a152e0 is 50, key is test_row_1/C:col10/1731971877797/Put/seqid=0 2024-11-18T23:17:59,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742448_1624 (size=9857) 2024-11-18T23:17:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T23:18:00,238 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/69ae9cc6ef754d11b9109859d2a152e0 2024-11-18T23:18:00,242 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/A/1dfd5aad8d0b46cb853a3c9ad84d0c44 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1dfd5aad8d0b46cb853a3c9ad84d0c44 2024-11-18T23:18:00,245 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1dfd5aad8d0b46cb853a3c9ad84d0c44, entries=100, sequenceid=562, filesize=22.0 K 2024-11-18T23:18:00,246 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/B/da6b8a65d6ea4a8fb6eb25161106e9ed as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/da6b8a65d6ea4a8fb6eb25161106e9ed 2024-11-18T23:18:00,249 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/da6b8a65d6ea4a8fb6eb25161106e9ed, entries=100, sequenceid=562, filesize=9.6 K 2024-11-18T23:18:00,249 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/.tmp/C/69ae9cc6ef754d11b9109859d2a152e0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/69ae9cc6ef754d11b9109859d2a152e0 2024-11-18T23:18:00,251 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/69ae9cc6ef754d11b9109859d2a152e0, entries=100, sequenceid=562, filesize=9.6 K 2024-11-18T23:18:00,252 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b28daae718a8c78c9acc9f3a87d1c727 in 1684ms, sequenceid=562, compaction requested=true 2024-11-18T23:18:00,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/702611b7ad334eb1bb4882b0c8f94ea7, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/de659c5cd4c64758ab62eb35703acabf, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/b63a0c04e2ac45489ab0b67017bd7f5c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/bd95ff50c69b4066a2b3bf0cc48f6cfb, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/4ce1b93b9f944023a358759ba6ed9b64, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae4a25e5181d4bf28c8a716df216378e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7338320fd74e45c9ab08339a6f55c69d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f534249813bc4af8bec1c6c1cbd923e0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/02060da2cf964cbdb0ddc5e984105d3a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/09f7eaf8105c423db96a95b89aae8b2c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ad7e79291aad471a995cd9b125f8b3ce, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/d29dd2c3af0841279a35fcb67644cc50, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7783f5a8f6b94dfe94711c8bb72a7ecc, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/48bfd3946f9f4da59fb03f75c8890a11, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/782fb73a6f8d4bf88c43e8f1558f1759, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/691cf79d72b54e899fba281e4a9f12ec, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/0e476e9166e1410a9fc07e2025f3d7a3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1ac8124d72994a778c82f117c6f5a18c, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ecb3ea4bd436427d85cc644e3e270a0b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ea26061fb2dc4a17a27ced121edb5796, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/85ede9b42a8a4d6ca1519a7e4b40eb75, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae240cee298e427085a08d25af9f1e03, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/238a856d41d848ff9faaf7f42158df07] to archive 2024-11-18T23:18:00,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:18:00,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/702611b7ad334eb1bb4882b0c8f94ea7 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/702611b7ad334eb1bb4882b0c8f94ea7 2024-11-18T23:18:00,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/de659c5cd4c64758ab62eb35703acabf to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/de659c5cd4c64758ab62eb35703acabf 2024-11-18T23:18:00,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/b63a0c04e2ac45489ab0b67017bd7f5c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/b63a0c04e2ac45489ab0b67017bd7f5c 2024-11-18T23:18:00,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/bd95ff50c69b4066a2b3bf0cc48f6cfb to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/bd95ff50c69b4066a2b3bf0cc48f6cfb 2024-11-18T23:18:00,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/eeb6da0c8b5f40f9ac8dc9c9f6e26279 2024-11-18T23:18:00,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/4ce1b93b9f944023a358759ba6ed9b64 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/4ce1b93b9f944023a358759ba6ed9b64 2024-11-18T23:18:00,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae4a25e5181d4bf28c8a716df216378e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae4a25e5181d4bf28c8a716df216378e 2024-11-18T23:18:00,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7338320fd74e45c9ab08339a6f55c69d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7338320fd74e45c9ab08339a6f55c69d 2024-11-18T23:18:00,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f534249813bc4af8bec1c6c1cbd923e0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f534249813bc4af8bec1c6c1cbd923e0 2024-11-18T23:18:00,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/02060da2cf964cbdb0ddc5e984105d3a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/02060da2cf964cbdb0ddc5e984105d3a 2024-11-18T23:18:00,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/09f7eaf8105c423db96a95b89aae8b2c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/09f7eaf8105c423db96a95b89aae8b2c 2024-11-18T23:18:00,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ad7e79291aad471a995cd9b125f8b3ce to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ad7e79291aad471a995cd9b125f8b3ce 2024-11-18T23:18:00,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/d29dd2c3af0841279a35fcb67644cc50 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/d29dd2c3af0841279a35fcb67644cc50 2024-11-18T23:18:00,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7783f5a8f6b94dfe94711c8bb72a7ecc to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/7783f5a8f6b94dfe94711c8bb72a7ecc 2024-11-18T23:18:00,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/48bfd3946f9f4da59fb03f75c8890a11 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/48bfd3946f9f4da59fb03f75c8890a11 2024-11-18T23:18:00,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/782fb73a6f8d4bf88c43e8f1558f1759 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/782fb73a6f8d4bf88c43e8f1558f1759 2024-11-18T23:18:00,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/691cf79d72b54e899fba281e4a9f12ec to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/691cf79d72b54e899fba281e4a9f12ec 2024-11-18T23:18:00,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/0e476e9166e1410a9fc07e2025f3d7a3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/0e476e9166e1410a9fc07e2025f3d7a3 2024-11-18T23:18:00,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1ac8124d72994a778c82f117c6f5a18c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1ac8124d72994a778c82f117c6f5a18c 2024-11-18T23:18:00,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ecb3ea4bd436427d85cc644e3e270a0b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ecb3ea4bd436427d85cc644e3e270a0b 2024-11-18T23:18:00,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ea26061fb2dc4a17a27ced121edb5796 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ea26061fb2dc4a17a27ced121edb5796 2024-11-18T23:18:00,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/85ede9b42a8a4d6ca1519a7e4b40eb75 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/85ede9b42a8a4d6ca1519a7e4b40eb75 2024-11-18T23:18:00,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae240cee298e427085a08d25af9f1e03 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/ae240cee298e427085a08d25af9f1e03 2024-11-18T23:18:00,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/238a856d41d848ff9faaf7f42158df07 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/238a856d41d848ff9faaf7f42158df07 2024-11-18T23:18:00,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/7e2b529cbbfc4743807319cbce795a32, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/97dd0be6dd92426698375bde35b7da7b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6c297c09c4a64c179c92f0af91d871cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3655b95405d044febf467cd77262e423, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/23ee8e71eaef4f9e820de6345f9654a0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/85ee7c821cb642f4b3018470e9edeb95, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/53e13288516140249010edfe8dd3328e, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93322b7ee4014b6181051ce435378f83, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93f4d9fc1c974b949d7ac44b8aaefd60, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/15a98214802149839d77fb0a913033b4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6cf7be076f5346308363faa5092223f9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/71cef26f5db742bebdae60d25eec1204, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f43168a7acbd43caa6f504c2e138fde0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f1e3c1bf7cd64c61b55758ab903aa5e2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ba3842927d3f486181eb7ba972f5b170, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/b101afc5162b47909ec7a340d11f4309, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/0541af3ca4d142749c15932ae558ced9, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/a8f886dfef904f72b8fca964778ef827, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/bdd70a70dd1c49c083e5a5c8d54ed077, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ef79277681c34b9da1d5c1db122be126, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/1fb0a43981d7454ca2f26464081e85f0, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f9fa07e8877744e9ae6ce1129f63d997, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/56c0d1a36a2f47e0907550f30fa63b2c] to archive 2024-11-18T23:18:00,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:18:00,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/c76f4c5dc9e04ae4bcc1abe86cd45b6a 2024-11-18T23:18:00,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/7e2b529cbbfc4743807319cbce795a32 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/7e2b529cbbfc4743807319cbce795a32 2024-11-18T23:18:00,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/97dd0be6dd92426698375bde35b7da7b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/97dd0be6dd92426698375bde35b7da7b 2024-11-18T23:18:00,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6c297c09c4a64c179c92f0af91d871cd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6c297c09c4a64c179c92f0af91d871cd 2024-11-18T23:18:00,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3655b95405d044febf467cd77262e423 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3655b95405d044febf467cd77262e423 2024-11-18T23:18:00,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/23ee8e71eaef4f9e820de6345f9654a0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/23ee8e71eaef4f9e820de6345f9654a0 2024-11-18T23:18:00,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/85ee7c821cb642f4b3018470e9edeb95 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/85ee7c821cb642f4b3018470e9edeb95 2024-11-18T23:18:00,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/53e13288516140249010edfe8dd3328e to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/53e13288516140249010edfe8dd3328e 2024-11-18T23:18:00,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93322b7ee4014b6181051ce435378f83 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93322b7ee4014b6181051ce435378f83 2024-11-18T23:18:00,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93f4d9fc1c974b949d7ac44b8aaefd60 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/93f4d9fc1c974b949d7ac44b8aaefd60 2024-11-18T23:18:00,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/15a98214802149839d77fb0a913033b4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/15a98214802149839d77fb0a913033b4 2024-11-18T23:18:00,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6cf7be076f5346308363faa5092223f9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/6cf7be076f5346308363faa5092223f9 2024-11-18T23:18:00,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/71cef26f5db742bebdae60d25eec1204 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/71cef26f5db742bebdae60d25eec1204 2024-11-18T23:18:00,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f43168a7acbd43caa6f504c2e138fde0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f43168a7acbd43caa6f504c2e138fde0 2024-11-18T23:18:00,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f1e3c1bf7cd64c61b55758ab903aa5e2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f1e3c1bf7cd64c61b55758ab903aa5e2 2024-11-18T23:18:00,287 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ba3842927d3f486181eb7ba972f5b170 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ba3842927d3f486181eb7ba972f5b170 2024-11-18T23:18:00,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/b101afc5162b47909ec7a340d11f4309 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/b101afc5162b47909ec7a340d11f4309 2024-11-18T23:18:00,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/0541af3ca4d142749c15932ae558ced9 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/0541af3ca4d142749c15932ae558ced9 2024-11-18T23:18:00,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/a8f886dfef904f72b8fca964778ef827 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/a8f886dfef904f72b8fca964778ef827 2024-11-18T23:18:00,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/bdd70a70dd1c49c083e5a5c8d54ed077 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/bdd70a70dd1c49c083e5a5c8d54ed077 2024-11-18T23:18:00,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ef79277681c34b9da1d5c1db122be126 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/ef79277681c34b9da1d5c1db122be126 2024-11-18T23:18:00,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/1fb0a43981d7454ca2f26464081e85f0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/1fb0a43981d7454ca2f26464081e85f0 2024-11-18T23:18:00,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f9fa07e8877744e9ae6ce1129f63d997 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/f9fa07e8877744e9ae6ce1129f63d997 2024-11-18T23:18:00,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/56c0d1a36a2f47e0907550f30fa63b2c to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/56c0d1a36a2f47e0907550f30fa63b2c 2024-11-18T23:18:00,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4fba964966774b769c7e1dc0f773f2da, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/5b6f49b99dd047faa568464c0a113dcb, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/3f47627eb9f74356b94f55c0d5b245a3, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/8fc7d25b519e4ed9b150224febd7bb69, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/15aeecf1f91d4f918788bb2de88aed58, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/835253ac3f434c3db2defbb448c4a94f, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/bc2000ca7f024232b9c398e5cddbf9d4, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/9da7221e4e1f4b18a528eb26d106e5cd, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/769eb3113b6744879611b575436291c8, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4beea62834af47d49732447a25797c8b, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/64e28894db1040e1ab59f62540c64e17, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ffe3ed010eee4cf3a3bf964b753fbc9a, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f1e693eb9377448aacd585ba835d86f2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b59384bc28eb4ab481ee1d079e629646, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c1aef64ecaac49e5928c2ca523e4ddf2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/02fc5841bbdc4de78969e0486865935d, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/55ac0c1b5bb847dbb9c169b3ffdd0e90, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/295e30e495df4de580f450e0a002fcf2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c2b6484e2eb446d79c74d2e997915790, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ef94bcc6b6a84980a86abe61e67f5282, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f709d57586b6452685eb4789c63128be, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/e5dcb60916f1484c9033b96a2d147659, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/0623809361964462b6ae4c1d8c9755e2, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/721f1f74303b44109fb16c6da47b4459] to archive 2024-11-18T23:18:00,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-18T23:18:00,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4fba964966774b769c7e1dc0f773f2da to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4fba964966774b769c7e1dc0f773f2da 2024-11-18T23:18:00,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/5b6f49b99dd047faa568464c0a113dcb to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/5b6f49b99dd047faa568464c0a113dcb 2024-11-18T23:18:00,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/3f47627eb9f74356b94f55c0d5b245a3 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/3f47627eb9f74356b94f55c0d5b245a3 2024-11-18T23:18:00,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/8fc7d25b519e4ed9b150224febd7bb69 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/8fc7d25b519e4ed9b150224febd7bb69 2024-11-18T23:18:00,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/15aeecf1f91d4f918788bb2de88aed58 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/15aeecf1f91d4f918788bb2de88aed58 2024-11-18T23:18:00,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/835253ac3f434c3db2defbb448c4a94f to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/835253ac3f434c3db2defbb448c4a94f 2024-11-18T23:18:00,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/bc2000ca7f024232b9c398e5cddbf9d4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/bc2000ca7f024232b9c398e5cddbf9d4 2024-11-18T23:18:00,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/9da7221e4e1f4b18a528eb26d106e5cd to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/9da7221e4e1f4b18a528eb26d106e5cd 2024-11-18T23:18:00,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/769eb3113b6744879611b575436291c8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/769eb3113b6744879611b575436291c8 2024-11-18T23:18:00,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4beea62834af47d49732447a25797c8b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/4beea62834af47d49732447a25797c8b 2024-11-18T23:18:00,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/64e28894db1040e1ab59f62540c64e17 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/64e28894db1040e1ab59f62540c64e17 2024-11-18T23:18:00,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ffe3ed010eee4cf3a3bf964b753fbc9a to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ffe3ed010eee4cf3a3bf964b753fbc9a 2024-11-18T23:18:00,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f1e693eb9377448aacd585ba835d86f2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f1e693eb9377448aacd585ba835d86f2 2024-11-18T23:18:00,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b59384bc28eb4ab481ee1d079e629646 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b59384bc28eb4ab481ee1d079e629646 2024-11-18T23:18:00,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c1aef64ecaac49e5928c2ca523e4ddf2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c1aef64ecaac49e5928c2ca523e4ddf2 2024-11-18T23:18:00,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/02fc5841bbdc4de78969e0486865935d to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/02fc5841bbdc4de78969e0486865935d 2024-11-18T23:18:00,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/55ac0c1b5bb847dbb9c169b3ffdd0e90 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/55ac0c1b5bb847dbb9c169b3ffdd0e90 2024-11-18T23:18:00,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/295e30e495df4de580f450e0a002fcf2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/295e30e495df4de580f450e0a002fcf2 2024-11-18T23:18:00,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c2b6484e2eb446d79c74d2e997915790 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/c2b6484e2eb446d79c74d2e997915790 2024-11-18T23:18:00,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ef94bcc6b6a84980a86abe61e67f5282 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/ef94bcc6b6a84980a86abe61e67f5282 2024-11-18T23:18:00,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f709d57586b6452685eb4789c63128be to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/f709d57586b6452685eb4789c63128be 2024-11-18T23:18:00,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/e5dcb60916f1484c9033b96a2d147659 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/e5dcb60916f1484c9033b96a2d147659 2024-11-18T23:18:00,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/0623809361964462b6ae4c1d8c9755e2 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/0623809361964462b6ae4c1d8c9755e2 2024-11-18T23:18:00,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/721f1f74303b44109fb16c6da47b4459 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/721f1f74303b44109fb16c6da47b4459 2024-11-18T23:18:00,361 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/recovered.edits/565.seqid, newMaxSeqId=565, maxSeqId=4 2024-11-18T23:18:00,362 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. 2024-11-18T23:18:00,362 DEBUG [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] regionserver.HRegion(1676): Region close journal for b28daae718a8c78c9acc9f3a87d1c727: Waiting for close lock at 1731971877969Running coprocessor pre-close hooks at 1731971877969Disabling compacts and flushes for region at 1731971877969Disabling writes for close at 1731971878568 (+599 ms)Obtaining lock to block concurrent updates at 1731971878568Preparing flush snapshotting stores in b28daae718a8c78c9acc9f3a87d1c727 at 1731971878568Finished memstore snapshotting TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731971878569 (+1 ms)Flushing stores of TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727. at 1731971878572 (+3 ms)Flushing b28daae718a8c78c9acc9f3a87d1c727/A: creating writer at 1731971878572Flushing mob file b28daae718a8c78c9acc9f3a87d1c727/A: appending metadata at 1731971878575 (+3 ms)Flushing mob file b28daae718a8c78c9acc9f3a87d1c727/A: closing flushed file at 1731971878575Flushing b28daae718a8c78c9acc9f3a87d1c727/A: appending metadata at 1731971878985 (+410 ms)Flushing b28daae718a8c78c9acc9f3a87d1c727/A: closing flushed file at 1731971878985Flushing b28daae718a8c78c9acc9f3a87d1c727/B: creating writer at 1731971879398 (+413 ms)Flushing b28daae718a8c78c9acc9f3a87d1c727/B: appending metadata at 1731971879401 (+3 ms)Flushing b28daae718a8c78c9acc9f3a87d1c727/B: closing flushed file at 1731971879401Flushing b28daae718a8c78c9acc9f3a87d1c727/C: creating writer at 1731971879830 (+429 ms)Flushing b28daae718a8c78c9acc9f3a87d1c727/C: appending metadata at 1731971879832 (+2 ms)Flushing b28daae718a8c78c9acc9f3a87d1c727/C: closing flushed file at 1731971879832Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@524feb15: reopening flushed file at 1731971880241 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@272e5187: reopening flushed file at 1731971880245 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37d7ff7a: reopening flushed file at 1731971880249 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for b28daae718a8c78c9acc9f3a87d1c727 in 1684ms, sequenceid=562, compaction requested=true at 1731971880252 (+3 ms)Writing region close event to WAL at 1731971880352 (+100 ms)Running coprocessor post-close hooks at 1731971880362 (+10 ms)Closed at 1731971880362 2024-11-18T23:18:00,364 INFO [RS_CLOSE_REGION-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_REGION, pid=155}] handler.UnassignRegionHandler(157): Closed b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:00,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=154 updating hbase:meta row=b28daae718a8c78c9acc9f3a87d1c727, regionState=CLOSED 2024-11-18T23:18:00,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE, hasLock=false; CloseRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 because future has completed 2024-11-18T23:18:00,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-18T23:18:00,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; CloseRegionProcedure b28daae718a8c78c9acc9f3a87d1c727, server=9d96596e55a9,32899,1731971713394 in 2.5580 sec 2024-11-18T23:18:00,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-18T23:18:00,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b28daae718a8c78c9acc9f3a87d1c727, UNASSIGN in 2.5620 sec 2024-11-18T23:18:00,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-18T23:18:00,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5660 sec 2024-11-18T23:18:00,383 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731971880382"}]},"ts":"1731971880382"} 2024-11-18T23:18:00,384 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-18T23:18:00,384 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-18T23:18:00,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5760 sec 2024-11-18T23:18:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-18T23:18:01,981 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:18:01,982 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-18T23:18:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:18:01,991 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=156, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:18:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T23:18:01,994 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=156, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:18:02,007 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,011 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C, FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/recovered.edits] 2024-11-18T23:18:02,014 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1dfd5aad8d0b46cb853a3c9ad84d0c44 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/1dfd5aad8d0b46cb853a3c9ad84d0c44 2024-11-18T23:18:02,019 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/5ac4cc0b1aaa47a39d81538378e7d28b to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/5ac4cc0b1aaa47a39d81538378e7d28b 2024-11-18T23:18:02,021 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/a2c4ba39bb7446989d220cc6fc1675a4 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/a2c4ba39bb7446989d220cc6fc1675a4 2024-11-18T23:18:02,022 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/c642fcd76615494d8ef5d47b64b53def to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/c642fcd76615494d8ef5d47b64b53def 2024-11-18T23:18:02,025 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f4d0f8fa0b314de1b8a49bf3b8f9a195 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/A/f4d0f8fa0b314de1b8a49bf3b8f9a195 2024-11-18T23:18:02,037 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/05a42a41d66c4779ba5d692cec634dab to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/05a42a41d66c4779ba5d692cec634dab 2024-11-18T23:18:02,044 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/13d44a22b5bf469fb4e1af2a713ce2f8 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/13d44a22b5bf469fb4e1af2a713ce2f8 2024-11-18T23:18:02,046 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3230e4cd7daa4f58950540d9b8545917 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/3230e4cd7daa4f58950540d9b8545917 2024-11-18T23:18:02,048 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/d59b98f4d56c4bcbb0db0abe752c2a85 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/d59b98f4d56c4bcbb0db0abe752c2a85 2024-11-18T23:18:02,050 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/da6b8a65d6ea4a8fb6eb25161106e9ed to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/B/da6b8a65d6ea4a8fb6eb25161106e9ed 2024-11-18T23:18:02,052 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/38227a72685d47b388990587c0586f46 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/38227a72685d47b388990587c0586f46 2024-11-18T23:18:02,054 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/69ae9cc6ef754d11b9109859d2a152e0 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/69ae9cc6ef754d11b9109859d2a152e0 2024-11-18T23:18:02,054 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b3ae505cb2864efaa3ec83b62f1ad513 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/b3ae505cb2864efaa3ec83b62f1ad513 2024-11-18T23:18:02,055 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/cbca8db655da4d09b91ecac8c148ab66 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/cbca8db655da4d09b91ecac8c148ab66 2024-11-18T23:18:02,056 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/eee0db6f381f4b868dd0de17197e4a69 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/C/eee0db6f381f4b868dd0de17197e4a69 2024-11-18T23:18:02,058 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/recovered.edits/565.seqid to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727/recovered.edits/565.seqid 2024-11-18T23:18:02,059 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/default/TestAcidGuarantees/b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,060 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-18T23:18:02,060 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T23:18:02,061 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-18T23:18:02,064 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111802f468bc83e84189a64d41bb8032d8e3_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111802f468bc83e84189a64d41bb8032d8e3_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,069 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111808fd491c041e415db4e5699fac2dfbf4_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111808fd491c041e415db4e5699fac2dfbf4_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,070 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180a55dea8adbb453e9bbcbcbf1ff0f690_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180a55dea8adbb453e9bbcbcbf1ff0f690_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,071 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180aba8bf3440645119d27f22469a24dba_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411180aba8bf3440645119d27f22469a24dba_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,072 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181d6d76518b054fec88b7474c090ac561_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411181d6d76518b054fec88b7474c090ac561_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,074 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183d66e201d74b4b569dd2fe8cb8a44929_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411183d66e201d74b4b569dd2fe8cb8a44929_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,076 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111842fda6666a0a4760bb813061c26ef5db_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111842fda6666a0a4760bb813061c26ef5db_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,077 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118453c49039c594ec3a18c029453723493_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118453c49039c594ec3a18c029453723493_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,079 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118479d1afe9bf84d778a4e46cc913b3ef4_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118479d1afe9bf84d778a4e46cc913b3ef4_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,080 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111850fb734dbdf147438f3c3c78647af301_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111850fb734dbdf147438f3c3c78647af301_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,081 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118575feb09b8e7429cb286c5b794d48aee_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118575feb09b8e7429cb286c5b794d48aee_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,083 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185bdb18cd3f224a7088326e802867b273_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411185bdb18cd3f224a7088326e802867b273_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,084 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118613b6ea22fd74f0eb9914f793879b545_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118613b6ea22fd74f0eb9914f793879b545_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,085 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186cd0da578efe4113bad14c16abfd44c4_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411186cd0da578efe4113bad14c16abfd44c4_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,086 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fcfb3798501478c826e0a79d749ee76_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fcfb3798501478c826e0a79d749ee76_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,087 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fed30c8abee48dabfbf4ee9356ffce8_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411188fed30c8abee48dabfbf4ee9356ffce8_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,087 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111893391db7c23b49de9b018525ded4d7d9_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111893391db7c23b49de9b018525ded4d7d9_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,088 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a945988badd34f11a007166eb5915cfa_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118a945988badd34f11a007166eb5915cfa_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,089 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b337d4bbb46a478bb3961475734bd7de_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118b337d4bbb46a478bb3961475734bd7de_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,090 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e3f4c3ea062a493a8f722794383c68af_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118e3f4c3ea062a493a8f722794383c68af_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,090 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118eaa301c821dc4b299a5b9f639ede7f3d_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118eaa301c821dc4b299a5b9f639ede7f3d_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,091 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ed2e6e2edc5844ac9084e8d0e2a3bf3f_b28daae718a8c78c9acc9f3a87d1c727 to hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241118ed2e6e2edc5844ac9084e8d0e2a3bf3f_b28daae718a8c78c9acc9f3a87d1c727 2024-11-18T23:18:02,092 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-18T23:18:02,094 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=156, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:18:02,096 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-18T23:18:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T23:18:02,099 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-18T23:18:02,101 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=156, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:18:02,101 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-18T23:18:02,101 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731971882101"}]},"ts":"9223372036854775807"} 2024-11-18T23:18:02,105 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-18T23:18:02,105 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b28daae718a8c78c9acc9f3a87d1c727, NAME => 'TestAcidGuarantees,,1731971854329.b28daae718a8c78c9acc9f3a87d1c727.', STARTKEY => '', ENDKEY => ''}] 2024-11-18T23:18:02,106 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-18T23:18:02,106 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731971882106"}]},"ts":"9223372036854775807"} 2024-11-18T23:18:02,109 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-18T23:18:02,110 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=156, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-18T23:18:02,112 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 128 msec 2024-11-18T23:18:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42339 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-18T23:18:02,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-18T23:18:02,308 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-18T23:18:02,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 250) - Thread LEAK? -, OpenFileDescriptor=457 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1658 (was 1623) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3664 (was 4351) 2024-11-18T23:18:02,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T23:18:02,323 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:18:02,323 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:18:02,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:18:02,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:18:02,323 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T23:18:02,323 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T23:18:02,323 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1201124139, stopped=false 2024-11-18T23:18:02,324 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9d96596e55a9,42339,1731971712494 2024-11-18T23:18:02,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:18:02,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T23:18:02,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:18:02,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:18:02,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:18:02,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:18:02,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T23:18:02,336 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T23:18:02,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:18:02,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:18:02,337 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9d96596e55a9,32899,1731971713394' ***** 2024-11-18T23:18:02,337 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T23:18:02,337 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T23:18:02,338 INFO [RS:0;9d96596e55a9:32899 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T23:18:02,338 INFO [RS:0;9d96596e55a9:32899 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T23:18:02,338 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(959): stopping server 9d96596e55a9,32899,1731971713394 2024-11-18T23:18:02,338 INFO [RS:0;9d96596e55a9:32899 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:18:02,338 INFO [RS:0;9d96596e55a9:32899 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9d96596e55a9:32899. 2024-11-18T23:18:02,338 DEBUG [RS:0;9d96596e55a9:32899 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T23:18:02,338 DEBUG [RS:0;9d96596e55a9:32899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:18:02,338 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T23:18:02,339 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T23:18:02,339 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T23:18:02,339 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T23:18:02,339 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T23:18:02,340 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T23:18:02,340 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T23:18:02,340 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:18:02,340 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T23:18:02,340 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T23:18:02,340 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T23:18:02,340 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T23:18:02,340 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T23:18:02,340 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-18T23:18:02,376 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/info/0c6f050f8681496a971a26ff3904d64e is 95, key is TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965./info:/1731971744988/DeleteFamily/seqid=0 2024-11-18T23:18:02,377 INFO [regionserver/9d96596e55a9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:18:02,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742449_1625 (size=6514) 2024-11-18T23:18:02,429 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/info/0c6f050f8681496a971a26ff3904d64e 2024-11-18T23:18:02,478 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/ns/fb580e193b274be287f1bfac7f98c33e is 93, key is TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965./ns:/1731971744893/DeleteFamily/seqid=0 2024-11-18T23:18:02,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742450_1626 (size=5906) 2024-11-18T23:18:02,541 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:18:02,741 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:18:02,898 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/ns/fb580e193b274be287f1bfac7f98c33e 2024-11-18T23:18:02,927 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/rep_barrier/a4b65313da6e4e678138987f31e0266b is 102, key is TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965./rep_barrier:/1731971744893/DeleteFamily/seqid=0 2024-11-18T23:18:02,927 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-18T23:18:02,941 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:18:02,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742451_1627 (size=6025) 2024-11-18T23:18:02,949 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/rep_barrier/a4b65313da6e4e678138987f31e0266b 2024-11-18T23:18:03,005 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/table/5f5c7f9e1f2e4d2f8ae9f09e8a7667bf is 96, key is TestAcidGuarantees,,1731971717798.14ab88ca9857fdcfce5643ad2ad0a965./table:/1731971744893/DeleteFamily/seqid=0 2024-11-18T23:18:03,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742452_1628 (size=5936) 2024-11-18T23:18:03,142 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:18:03,288 INFO [regionserver/9d96596e55a9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-18T23:18:03,288 INFO [regionserver/9d96596e55a9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-18T23:18:03,342 DEBUG [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T23:18:03,452 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/table/5f5c7f9e1f2e4d2f8ae9f09e8a7667bf 2024-11-18T23:18:03,457 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/info/0c6f050f8681496a971a26ff3904d64e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/info/0c6f050f8681496a971a26ff3904d64e 2024-11-18T23:18:03,460 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/info/0c6f050f8681496a971a26ff3904d64e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-18T23:18:03,462 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/ns/fb580e193b274be287f1bfac7f98c33e as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/ns/fb580e193b274be287f1bfac7f98c33e 2024-11-18T23:18:03,465 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/ns/fb580e193b274be287f1bfac7f98c33e, entries=8, sequenceid=90, filesize=5.8 K 2024-11-18T23:18:03,466 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/rep_barrier/a4b65313da6e4e678138987f31e0266b as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/rep_barrier/a4b65313da6e4e678138987f31e0266b 2024-11-18T23:18:03,469 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/rep_barrier/a4b65313da6e4e678138987f31e0266b, entries=6, sequenceid=90, filesize=5.9 K 2024-11-18T23:18:03,470 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/.tmp/table/5f5c7f9e1f2e4d2f8ae9f09e8a7667bf as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/table/5f5c7f9e1f2e4d2f8ae9f09e8a7667bf 2024-11-18T23:18:03,474 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/table/5f5c7f9e1f2e4d2f8ae9f09e8a7667bf, entries=7, sequenceid=90, filesize=5.8 K 2024-11-18T23:18:03,475 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1135ms, sequenceid=90, compaction requested=false 2024-11-18T23:18:03,492 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-18T23:18:03,493 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T23:18:03,493 INFO [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T23:18:03,493 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731971882340Running coprocessor pre-close hooks at 1731971882340Disabling compacts and flushes for region at 1731971882340Disabling writes for close at 1731971882340Obtaining lock to block concurrent updates at 1731971882340Preparing flush snapshotting stores in 1588230740 at 1731971882340Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731971882341 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731971882342 (+1 ms)Flushing 1588230740/info: creating writer at 1731971882342Flushing 1588230740/info: appending metadata at 1731971882375 (+33 ms)Flushing 1588230740/info: closing flushed file at 1731971882375Flushing 1588230740/ns: creating writer at 1731971882442 (+67 ms)Flushing 1588230740/ns: appending metadata at 1731971882477 (+35 ms)Flushing 1588230740/ns: closing flushed file at 1731971882477Flushing 1588230740/rep_barrier: creating writer at 1731971882902 (+425 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731971882927 (+25 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731971882927Flushing 1588230740/table: creating writer at 1731971882961 (+34 ms)Flushing 1588230740/table: appending metadata at 1731971883005 (+44 ms)Flushing 1588230740/table: closing flushed file at 1731971883005Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57807471: reopening flushed file at 1731971883456 (+451 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6adca4b6: reopening flushed file at 1731971883461 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60695939: reopening flushed file at 1731971883465 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33b2fe9b: reopening flushed file at 1731971883470 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1135ms, sequenceid=90, compaction requested=false at 1731971883475 (+5 ms)Writing region close event to WAL at 1731971883488 (+13 ms)Running coprocessor post-close hooks at 1731971883493 (+5 ms)Closed at 1731971883493 2024-11-18T23:18:03,493 DEBUG [RS_CLOSE_META-regionserver/9d96596e55a9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T23:18:03,543 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(976): stopping server 9d96596e55a9,32899,1731971713394; all regions closed. 2024-11-18T23:18:03,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741834_1010 (size=25344) 2024-11-18T23:18:03,573 DEBUG [RS:0;9d96596e55a9:32899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/oldWALs 2024-11-18T23:18:03,576 INFO [RS:0;9d96596e55a9:32899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9d96596e55a9%2C32899%2C1731971713394.meta:.meta(num 1731971717362) 2024-11-18T23:18:03,596 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/WALs/9d96596e55a9,32899,1731971713394/9d96596e55a9%2C32899%2C1731971713394.1731971716450 not finished, retry = 0 2024-11-18T23:18:03,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741832_1008 (size=20711854) 2024-11-18T23:18:03,699 DEBUG [RS:0;9d96596e55a9:32899 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/oldWALs 2024-11-18T23:18:03,699 INFO [RS:0;9d96596e55a9:32899 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9d96596e55a9%2C32899%2C1731971713394:(num 1731971716450) 2024-11-18T23:18:03,699 DEBUG [RS:0;9d96596e55a9:32899 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T23:18:03,699 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T23:18:03,700 INFO [RS:0;9d96596e55a9:32899 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:18:03,700 INFO [RS:0;9d96596e55a9:32899 {}] hbase.ChoreService(370): Chore service for: regionserver/9d96596e55a9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T23:18:03,700 INFO [RS:0;9d96596e55a9:32899 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:18:03,700 INFO [regionserver/9d96596e55a9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:18:03,702 INFO [RS:0;9d96596e55a9:32899 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32899 2024-11-18T23:18:03,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T23:18:03,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9d96596e55a9,32899,1731971713394 2024-11-18T23:18:03,714 INFO [RS:0;9d96596e55a9:32899 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:18:03,715 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9d96596e55a9,32899,1731971713394] 2024-11-18T23:18:03,735 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9d96596e55a9,32899,1731971713394 already deleted, retry=false 2024-11-18T23:18:03,735 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9d96596e55a9,32899,1731971713394 expired; onlineServers=0 2024-11-18T23:18:03,735 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9d96596e55a9,42339,1731971712494' ***** 2024-11-18T23:18:03,735 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T23:18:03,736 INFO [M:0;9d96596e55a9:42339 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T23:18:03,736 INFO [M:0;9d96596e55a9:42339 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T23:18:03,736 DEBUG [M:0;9d96596e55a9:42339 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T23:18:03,736 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T23:18:03,736 DEBUG [M:0;9d96596e55a9:42339 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T23:18:03,736 DEBUG [master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.large.0-1731971716089 {}] cleaner.HFileCleaner(306): Exit Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.large.0-1731971716089,5,FailOnTimeoutGroup] 2024-11-18T23:18:03,736 INFO [M:0;9d96596e55a9:42339 {}] hbase.ChoreService(370): Chore service for: master/9d96596e55a9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T23:18:03,736 DEBUG [master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.small.0-1731971716090 {}] cleaner.HFileCleaner(306): Exit Thread[master/9d96596e55a9:0:becomeActiveMaster-HFileCleaner.small.0-1731971716090,5,FailOnTimeoutGroup] 2024-11-18T23:18:03,736 INFO [M:0;9d96596e55a9:42339 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T23:18:03,737 DEBUG [M:0;9d96596e55a9:42339 {}] master.HMaster(1795): Stopping service threads 2024-11-18T23:18:03,737 INFO [M:0;9d96596e55a9:42339 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T23:18:03,737 INFO [M:0;9d96596e55a9:42339 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T23:18:03,737 ERROR [M:0;9d96596e55a9:42339 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:39239 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39239,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-18T23:18:03,738 INFO [M:0;9d96596e55a9:42339 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T23:18:03,739 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T23:18:03,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T23:18:03,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T23:18:03,762 DEBUG [M:0;9d96596e55a9:42339 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-18T23:18:03,762 DEBUG [M:0;9d96596e55a9:42339 {}] master.ActiveMasterManager(353): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-18T23:18:03,763 INFO [M:0;9d96596e55a9:42339 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/.lastflushedseqids 2024-11-18T23:18:03,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742453_1629 (size=72) 2024-11-18T23:18:03,784 INFO [M:0;9d96596e55a9:42339 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T23:18:03,785 INFO [M:0;9d96596e55a9:42339 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T23:18:03,785 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T23:18:03,785 INFO [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:18:03,785 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:18:03,785 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T23:18:03,785 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:18:03,785 INFO [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=760.25 KB heapSize=930.63 KB 2024-11-18T23:18:03,807 DEBUG [M:0;9d96596e55a9:42339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/af662913e8b14826926eebd9d5b9efb0 is 82, key is hbase:meta,,1/info:regioninfo/1731971717460/Put/seqid=0 2024-11-18T23:18:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742454_1630 (size=5672) 2024-11-18T23:18:03,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:18:03,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32899-0x101503afdd90001, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:18:03,825 INFO [M:0;9d96596e55a9:42339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2111 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/af662913e8b14826926eebd9d5b9efb0 2024-11-18T23:18:03,826 INFO [RS:0;9d96596e55a9:32899 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:18:03,826 INFO [RS:0;9d96596e55a9:32899 {}] regionserver.HRegionServer(1031): Exiting; stopping=9d96596e55a9,32899,1731971713394; zookeeper connection closed. 2024-11-18T23:18:03,826 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@476babce {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@476babce 2024-11-18T23:18:03,827 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-18T23:18:03,871 DEBUG [M:0;9d96596e55a9:42339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2a190902dba84e1ab50fbd266aa1c7be is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x85/proc:d/1731971856568/Put/seqid=0 2024-11-18T23:18:03,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742455_1631 (size=42909) 2024-11-18T23:18:03,895 INFO [M:0;9d96596e55a9:42339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=759.69 KB at sequenceid=2111 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2a190902dba84e1ab50fbd266aa1c7be 2024-11-18T23:18:03,908 INFO [M:0;9d96596e55a9:42339 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2a190902dba84e1ab50fbd266aa1c7be 2024-11-18T23:18:03,938 DEBUG [M:0;9d96596e55a9:42339 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f762a78b3d2048c183e14fd33d44c507 is 69, key is 9d96596e55a9,32899,1731971713394/rs:state/1731971716184/Put/seqid=0 2024-11-18T23:18:03,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073742456_1632 (size=5156) 2024-11-18T23:18:04,388 INFO [M:0;9d96596e55a9:42339 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2111 (bloomFilter=true), to=hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f762a78b3d2048c183e14fd33d44c507 2024-11-18T23:18:04,393 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/af662913e8b14826926eebd9d5b9efb0 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/af662913e8b14826926eebd9d5b9efb0 2024-11-18T23:18:04,398 INFO [M:0;9d96596e55a9:42339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/af662913e8b14826926eebd9d5b9efb0, entries=8, sequenceid=2111, filesize=5.5 K 2024-11-18T23:18:04,399 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2a190902dba84e1ab50fbd266aa1c7be as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2a190902dba84e1ab50fbd266aa1c7be 2024-11-18T23:18:04,403 INFO [M:0;9d96596e55a9:42339 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2a190902dba84e1ab50fbd266aa1c7be 2024-11-18T23:18:04,403 INFO [M:0;9d96596e55a9:42339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2a190902dba84e1ab50fbd266aa1c7be, entries=156, sequenceid=2111, filesize=41.9 K 2024-11-18T23:18:04,404 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f762a78b3d2048c183e14fd33d44c507 as hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f762a78b3d2048c183e14fd33d44c507 2024-11-18T23:18:04,409 INFO [M:0;9d96596e55a9:42339 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39239/user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f762a78b3d2048c183e14fd33d44c507, entries=1, sequenceid=2111, filesize=5.0 K 2024-11-18T23:18:04,410 INFO [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(3140): Finished flush of dataSize ~760.25 KB/778493, heapSize ~930.33 KB/952656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 625ms, sequenceid=2111, compaction requested=false 2024-11-18T23:18:04,437 INFO [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T23:18:04,437 DEBUG [M:0;9d96596e55a9:42339 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731971883785Disabling compacts and flushes for region at 1731971883785Disabling writes for close at 1731971883785Obtaining lock to block concurrent updates at 1731971883785Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731971883785Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=778493, getHeapSize=952896, getOffHeapSize=0, getCellsCount=2292 at 1731971883786 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731971883786Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731971883786Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731971883807 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731971883807Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731971883830 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731971883870 (+40 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731971883870Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731971883908 (+38 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731971883937 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731971883937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d933c42: reopening flushed file at 1731971884392 (+455 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@325c64dd: reopening flushed file at 1731971884398 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26bca6d5: reopening flushed file at 1731971884404 (+6 ms)Finished flush of dataSize ~760.25 KB/778493, heapSize ~930.33 KB/952656, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 625ms, sequenceid=2111, compaction requested=false at 1731971884410 (+6 ms)Writing region close event to WAL at 1731971884437 (+27 ms)Closed at 1731971884437 2024-11-18T23:18:04,451 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/5eeaa642-aa14-a064-1764-954c4215edff/MasterData/WALs/9d96596e55a9,42339,1731971712494/9d96596e55a9%2C42339%2C1731971712494.1731971715060 not finished, retry = 0 2024-11-18T23:18:04,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43067 is added to blk_1073741830_1006 (size=916851) 2024-11-18T23:18:04,552 INFO [M:0;9d96596e55a9:42339 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T23:18:04,552 INFO [M:0;9d96596e55a9:42339 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42339 2024-11-18T23:18:04,552 INFO [M:0;9d96596e55a9:42339 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T23:18:04,554 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T23:18:04,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:18:04,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42339-0x101503afdd90000, quorum=127.0.0.1:56682, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T23:18:04,663 INFO [M:0;9d96596e55a9:42339 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T23:18:04,678 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-18T23:18:04,685 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T23:18:04,686 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T23:18:04,686 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T23:18:04,686 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/hadoop.log.dir/,STOPPED} 2024-11-18T23:18:04,695 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-18T23:18:04,695 WARN [BP-2062329126-172.17.0.2-1731971708035 heartbeating to localhost/127.0.0.1:39239 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-18T23:18:04,695 WARN [BP-2062329126-172.17.0.2-1731971708035 heartbeating to localhost/127.0.0.1:39239 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2062329126-172.17.0.2-1731971708035 (Datanode Uuid 240527c7-c924-4f5d-b701-50d98ed6d905) service to localhost/127.0.0.1:39239 2024-11-18T23:18:04,695 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-18T23:18:04,697 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/data/data1/current/BP-2062329126-172.17.0.2-1731971708035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:18:04,697 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-18T23:18:04,698 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/cluster_d550a998-2efc-0931-fefa-548cfd9e0771/data/data2/current/BP-2062329126-172.17.0.2-1731971708035 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-18T23:18:04,710 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-18T23:18:04,713 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-18T23:18:04,713 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-18T23:18:04,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-18T23:18:04,714 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/05981ac3-0026-7b96-f1cc-7764756bbd5f/hadoop.log.dir/,STOPPED} 2024-11-18T23:18:04,743 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T23:18:05,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down